{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "在基础作业中分析了结果不好的原因，首先先和之前一样导入数据，再一步一步改善神经网络结构看看对之前造成的影响"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-2-8ef614dae8f3>:1: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From D:\\Users\\Administrator\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From D:\\Users\\Administrator\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting ./train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From D:\\Users\\Administrator\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting ./train-labels-idx1-ubyte.gz\n",
      "Extracting ./t10k-images-idx3-ubyte.gz\n",
      "Extracting ./t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From D:\\Users\\Administrator\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "mnist = input_data.read_data_sets(\"./\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = tf.placeholder(\"float\", [None, 784])#定义输入，输入数据是图片的784列一维数据，定义一个占位符，后面把数据输入进来\n",
    "y = tf.placeholder(\"int64\", [None])#输出数据自然是整数型的标签\n",
    "learning_rate = tf.placeholder(\"float\")#定义学习率\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def initialize(shape, stddev=0.1):#定义初始化函数。\n",
    "  return tf.truncated_normal(shape, stddev=0.1)#用truncated_normal函数进行初始化，truncated_normal是截断正态分布，会删除大于2个stddev的x值\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From D:\\Users\\Administrator\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n"
     ]
    }
   ],
   "source": [
    "L1_units_count = 100 #第一层神经元的个数\n",
    "W_1 = tf.Variable(initialize([784, L1_units_count])) #variable主要用于数据存储,在计算图的运算过程中，其值会一直保存到程序运行结束\n",
    "b_1 = tf.Variable(initialize([L1_units_count]))#比如神经网络中的权重和bias等，在训练过后，总是希望这些参数能够保存下来，而不是直接就消失了，所以这个时候要用到Variable\n",
    "logits_1 = tf.matmul(x, W_1) + b_1 #logits函数，即y=wx+b在tensorflow中计算图的定义方法\n",
    "output_1 = tf.nn.relu(logits_1) #激活函数为relu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "L2_units_count = 10 #最终输出的结果总过有10个\n",
    "W_2 = tf.Variable(initialize([L1_units_count, L2_units_count]))\n",
    "b_2 = tf.Variable(initialize([L2_units_count]))\n",
    "logits_2 = tf.matmul(output_1, W_2) + b_2  \n",
    "logits = logits_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "cross_entropy_loss = tf.reduce_mean(\n",
    "    tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=y))#损失函数是交叉熵损失\n",
    "\n",
    "\n",
    "batch_size = 100 #一轮喂入的数据为100个\n",
    "trainig_step = 20000 #训练10000个step\n",
    "\n",
    "saver = tf.train.Saver()#模型保存\n",
    "pred = tf.nn.softmax(logits)# 原始logit用softmax转换一下\n",
    "correct_pred = tf.equal(tf.argmax(pred, 1), y)# 预测的准确率\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))#取平均值得到最终的准确率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.train.GradientDescentOptimizer( #定义优化方法为梯度下降\n",
    "    learning_rate=learning_rate).minimize(cross_entropy_loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "增大训练轮数，增加batchsize,增加学习率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 100 training steps, the loss is 0.395158, the validation accuracy is 0.8824\n",
      "after 200 training steps, the loss is 0.224569, the validation accuracy is 0.9236\n",
      "after 300 training steps, the loss is 0.219436, the validation accuracy is 0.9432\n",
      "after 400 training steps, the loss is 0.154907, the validation accuracy is 0.9514\n",
      "after 500 training steps, the loss is 0.204463, the validation accuracy is 0.9544\n",
      "after 600 training steps, the loss is 0.135707, the validation accuracy is 0.9596\n",
      "after 700 training steps, the loss is 0.0882057, the validation accuracy is 0.9634\n",
      "after 800 training steps, the loss is 0.213962, the validation accuracy is 0.9658\n",
      "after 900 training steps, the loss is 0.128783, the validation accuracy is 0.9654\n",
      "after 1000 training steps, the loss is 0.179972, the validation accuracy is 0.9694\n",
      "after 1100 training steps, the loss is 0.147627, the validation accuracy is 0.9672\n",
      "after 1200 training steps, the loss is 0.201057, the validation accuracy is 0.9676\n",
      "after 1300 training steps, the loss is 0.176951, the validation accuracy is 0.9646\n",
      "after 1400 training steps, the loss is 0.0624232, the validation accuracy is 0.972\n",
      "after 1500 training steps, the loss is 0.110396, the validation accuracy is 0.9736\n",
      "after 1600 training steps, the loss is 0.036419, the validation accuracy is 0.9724\n",
      "after 1700 training steps, the loss is 0.0772009, the validation accuracy is 0.9736\n",
      "after 1800 training steps, the loss is 0.0502774, the validation accuracy is 0.9728\n",
      "after 1900 training steps, the loss is 0.109493, the validation accuracy is 0.971\n",
      "after 2000 training steps, the loss is 0.0328107, the validation accuracy is 0.9704\n",
      "after 2100 training steps, the loss is 0.0658256, the validation accuracy is 0.9762\n",
      "after 2200 training steps, the loss is 0.0920314, the validation accuracy is 0.9748\n",
      "after 2300 training steps, the loss is 0.0960106, the validation accuracy is 0.977\n",
      "after 2400 training steps, the loss is 0.0851095, the validation accuracy is 0.9722\n",
      "after 2500 training steps, the loss is 0.0608904, the validation accuracy is 0.976\n",
      "after 2600 training steps, the loss is 0.118139, the validation accuracy is 0.9702\n",
      "after 2700 training steps, the loss is 0.0215535, the validation accuracy is 0.9778\n",
      "after 2800 training steps, the loss is 0.0470353, the validation accuracy is 0.9778\n",
      "after 2900 training steps, the loss is 0.0826708, the validation accuracy is 0.976\n",
      "after 3000 training steps, the loss is 0.019119, the validation accuracy is 0.9762\n",
      "after 3100 training steps, the loss is 0.0267803, the validation accuracy is 0.975\n",
      "after 3200 training steps, the loss is 0.0359948, the validation accuracy is 0.9772\n",
      "after 3300 training steps, the loss is 0.0458118, the validation accuracy is 0.9776\n",
      "after 3400 training steps, the loss is 0.042568, the validation accuracy is 0.9772\n",
      "after 3500 training steps, the loss is 0.0695196, the validation accuracy is 0.9762\n",
      "after 3600 training steps, the loss is 0.0422204, the validation accuracy is 0.9758\n",
      "after 3700 training steps, the loss is 0.0140184, the validation accuracy is 0.9758\n",
      "after 3800 training steps, the loss is 0.074325, the validation accuracy is 0.9764\n",
      "after 3900 training steps, the loss is 0.0432621, the validation accuracy is 0.9774\n",
      "after 4000 training steps, the loss is 0.0159957, the validation accuracy is 0.9768\n",
      "after 4100 training steps, the loss is 0.0301126, the validation accuracy is 0.9752\n",
      "after 4200 training steps, the loss is 0.0875354, the validation accuracy is 0.9774\n",
      "after 4300 training steps, the loss is 0.00898847, the validation accuracy is 0.9782\n",
      "after 4400 training steps, the loss is 0.0197379, the validation accuracy is 0.9778\n",
      "after 4500 training steps, the loss is 0.0430931, the validation accuracy is 0.976\n",
      "after 4600 training steps, the loss is 0.04519, the validation accuracy is 0.9772\n",
      "after 4700 training steps, the loss is 0.0267111, the validation accuracy is 0.9776\n",
      "after 4800 training steps, the loss is 0.0147128, the validation accuracy is 0.9758\n",
      "after 4900 training steps, the loss is 0.0427254, the validation accuracy is 0.978\n",
      "after 5000 training steps, the loss is 0.00820091, the validation accuracy is 0.9784\n",
      "after 5100 training steps, the loss is 0.0596866, the validation accuracy is 0.9764\n",
      "after 5200 training steps, the loss is 0.034854, the validation accuracy is 0.976\n",
      "after 5300 training steps, the loss is 0.00743321, the validation accuracy is 0.9802\n",
      "after 5400 training steps, the loss is 0.00466727, the validation accuracy is 0.9758\n",
      "after 5500 training steps, the loss is 0.0225385, the validation accuracy is 0.978\n",
      "after 5600 training steps, the loss is 0.0122286, the validation accuracy is 0.9796\n",
      "after 5700 training steps, the loss is 0.0158211, the validation accuracy is 0.9798\n",
      "after 5800 training steps, the loss is 0.0142393, the validation accuracy is 0.9762\n",
      "after 5900 training steps, the loss is 0.0248865, the validation accuracy is 0.9762\n",
      "after 6000 training steps, the loss is 0.0155831, the validation accuracy is 0.9788\n",
      "after 6100 training steps, the loss is 0.0154096, the validation accuracy is 0.9772\n",
      "after 6200 training steps, the loss is 0.0145636, the validation accuracy is 0.9788\n",
      "after 6300 training steps, the loss is 0.0112166, the validation accuracy is 0.977\n",
      "after 6400 training steps, the loss is 0.0372065, the validation accuracy is 0.9766\n",
      "after 6500 training steps, the loss is 0.028834, the validation accuracy is 0.9784\n",
      "after 6600 training steps, the loss is 0.0187872, the validation accuracy is 0.979\n",
      "after 6700 training steps, the loss is 0.0246038, the validation accuracy is 0.9808\n",
      "after 6800 training steps, the loss is 0.0198788, the validation accuracy is 0.9788\n",
      "after 6900 training steps, the loss is 0.0141204, the validation accuracy is 0.9782\n",
      "after 7000 training steps, the loss is 0.00795221, the validation accuracy is 0.9786\n",
      "after 7100 training steps, the loss is 0.0147198, the validation accuracy is 0.98\n",
      "after 7200 training steps, the loss is 0.0112666, the validation accuracy is 0.9802\n",
      "after 7300 training steps, the loss is 0.00932293, the validation accuracy is 0.9804\n",
      "after 7400 training steps, the loss is 0.00841726, the validation accuracy is 0.9778\n",
      "after 7500 training steps, the loss is 0.0168812, the validation accuracy is 0.9778\n",
      "after 7600 training steps, the loss is 0.00328697, the validation accuracy is 0.9774\n",
      "after 7700 training steps, the loss is 0.0232994, the validation accuracy is 0.9772\n",
      "after 7800 training steps, the loss is 0.0158803, the validation accuracy is 0.9786\n",
      "after 7900 training steps, the loss is 0.00849926, the validation accuracy is 0.9788\n",
      "after 8000 training steps, the loss is 0.00116685, the validation accuracy is 0.9782\n",
      "after 8100 training steps, the loss is 0.00950975, the validation accuracy is 0.9778\n",
      "after 8200 training steps, the loss is 0.0143986, the validation accuracy is 0.9796\n",
      "after 8300 training steps, the loss is 0.00264803, the validation accuracy is 0.9796\n",
      "after 8400 training steps, the loss is 0.0124909, the validation accuracy is 0.9816\n",
      "after 8500 training steps, the loss is 0.00587878, the validation accuracy is 0.9794\n",
      "after 8600 training steps, the loss is 0.00865365, the validation accuracy is 0.9808\n",
      "after 8700 training steps, the loss is 0.00221029, the validation accuracy is 0.9796\n",
      "after 8800 training steps, the loss is 0.0131827, the validation accuracy is 0.9788\n",
      "after 8900 training steps, the loss is 0.00606475, the validation accuracy is 0.9794\n",
      "after 9000 training steps, the loss is 0.0478321, the validation accuracy is 0.9794\n",
      "after 9100 training steps, the loss is 0.00693441, the validation accuracy is 0.9806\n",
      "after 9200 training steps, the loss is 0.00292817, the validation accuracy is 0.9796\n",
      "after 9300 training steps, the loss is 0.00758287, the validation accuracy is 0.9798\n",
      "after 9400 training steps, the loss is 0.00177407, the validation accuracy is 0.9796\n",
      "after 9500 training steps, the loss is 0.00602816, the validation accuracy is 0.9794\n",
      "after 9600 training steps, the loss is 0.0069913, the validation accuracy is 0.9784\n",
      "after 9700 training steps, the loss is 0.020099, the validation accuracy is 0.9788\n",
      "after 9800 training steps, the loss is 0.00447847, the validation accuracy is 0.9792\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 9900 training steps, the loss is 0.00450905, the validation accuracy is 0.9804\n",
      "after 10000 training steps, the loss is 0.0119443, the validation accuracy is 0.9786\n",
      "after 10100 training steps, the loss is 0.00781987, the validation accuracy is 0.9792\n",
      "after 10200 training steps, the loss is 0.00177962, the validation accuracy is 0.9786\n",
      "after 10300 training steps, the loss is 0.00505454, the validation accuracy is 0.9808\n",
      "after 10400 training steps, the loss is 0.00133532, the validation accuracy is 0.9814\n",
      "after 10500 training steps, the loss is 0.00159774, the validation accuracy is 0.9804\n",
      "after 10600 training steps, the loss is 0.00727341, the validation accuracy is 0.9794\n",
      "after 10700 training steps, the loss is 0.00256019, the validation accuracy is 0.9784\n",
      "after 10800 training steps, the loss is 0.00288815, the validation accuracy is 0.977\n",
      "after 10900 training steps, the loss is 0.00631208, the validation accuracy is 0.98\n",
      "after 11000 training steps, the loss is 0.00165443, the validation accuracy is 0.98\n",
      "after 11100 training steps, the loss is 0.00326899, the validation accuracy is 0.9802\n",
      "after 11200 training steps, the loss is 0.00557655, the validation accuracy is 0.9798\n",
      "after 11300 training steps, the loss is 0.00352476, the validation accuracy is 0.9804\n",
      "after 11400 training steps, the loss is 0.0030725, the validation accuracy is 0.9796\n",
      "after 11500 training steps, the loss is 0.00618703, the validation accuracy is 0.9796\n",
      "after 11600 training steps, the loss is 0.00371545, the validation accuracy is 0.9802\n",
      "after 11700 training steps, the loss is 0.00357829, the validation accuracy is 0.9798\n",
      "after 11800 training steps, the loss is 0.00122401, the validation accuracy is 0.9806\n",
      "after 11900 training steps, the loss is 0.0132629, the validation accuracy is 0.9796\n",
      "after 12000 training steps, the loss is 0.00629569, the validation accuracy is 0.981\n",
      "after 12100 training steps, the loss is 0.00349192, the validation accuracy is 0.9796\n",
      "after 12200 training steps, the loss is 0.00584431, the validation accuracy is 0.9798\n",
      "after 12300 training steps, the loss is 0.00290266, the validation accuracy is 0.981\n",
      "after 12400 training steps, the loss is 0.00599905, the validation accuracy is 0.9804\n",
      "after 12500 training steps, the loss is 0.00335059, the validation accuracy is 0.9796\n",
      "after 12600 training steps, the loss is 0.00490798, the validation accuracy is 0.9802\n",
      "after 12700 training steps, the loss is 0.00360567, the validation accuracy is 0.9804\n",
      "after 12800 training steps, the loss is 0.00323787, the validation accuracy is 0.9798\n",
      "after 12900 training steps, the loss is 0.00258784, the validation accuracy is 0.979\n",
      "after 13000 training steps, the loss is 0.00872822, the validation accuracy is 0.9806\n",
      "after 13100 training steps, the loss is 0.00451237, the validation accuracy is 0.9796\n",
      "after 13200 training steps, the loss is 0.00188082, the validation accuracy is 0.9798\n",
      "after 13300 training steps, the loss is 0.00228199, the validation accuracy is 0.9806\n",
      "after 13400 training steps, the loss is 0.00365621, the validation accuracy is 0.9804\n",
      "after 13500 training steps, the loss is 0.00169801, the validation accuracy is 0.9796\n",
      "after 13600 training steps, the loss is 0.000537205, the validation accuracy is 0.981\n",
      "after 13700 training steps, the loss is 0.00190687, the validation accuracy is 0.9806\n",
      "after 13800 training steps, the loss is 0.00120187, the validation accuracy is 0.9802\n",
      "after 13900 training steps, the loss is 0.00144496, the validation accuracy is 0.98\n",
      "after 14000 training steps, the loss is 0.000934765, the validation accuracy is 0.9814\n",
      "after 14100 training steps, the loss is 0.00746585, the validation accuracy is 0.978\n",
      "after 14200 training steps, the loss is 0.000628689, the validation accuracy is 0.9802\n",
      "after 14300 training steps, the loss is 0.000602772, the validation accuracy is 0.98\n",
      "after 14400 training steps, the loss is 0.00181304, the validation accuracy is 0.9794\n",
      "after 14500 training steps, the loss is 0.00273293, the validation accuracy is 0.9806\n",
      "after 14600 training steps, the loss is 0.00180861, the validation accuracy is 0.9802\n",
      "after 14700 training steps, the loss is 0.00277182, the validation accuracy is 0.981\n",
      "after 14800 training steps, the loss is 0.00112255, the validation accuracy is 0.9806\n",
      "after 14900 training steps, the loss is 0.00133308, the validation accuracy is 0.9804\n",
      "after 15000 training steps, the loss is 0.00225772, the validation accuracy is 0.98\n",
      "after 15100 training steps, the loss is 0.00207691, the validation accuracy is 0.9806\n",
      "after 15200 training steps, the loss is 0.00181621, the validation accuracy is 0.9806\n",
      "after 15300 training steps, the loss is 0.000994547, the validation accuracy is 0.9816\n",
      "after 15400 training steps, the loss is 0.00116119, the validation accuracy is 0.9806\n",
      "after 15500 training steps, the loss is 0.0022651, the validation accuracy is 0.9802\n",
      "after 15600 training steps, the loss is 0.00128972, the validation accuracy is 0.9802\n",
      "after 15700 training steps, the loss is 0.00464712, the validation accuracy is 0.98\n",
      "after 15800 training steps, the loss is 0.0014353, the validation accuracy is 0.9802\n",
      "after 15900 training steps, the loss is 0.000523576, the validation accuracy is 0.9808\n",
      "after 16000 training steps, the loss is 0.00316262, the validation accuracy is 0.9804\n",
      "after 16100 training steps, the loss is 0.00342403, the validation accuracy is 0.9818\n",
      "after 16200 training steps, the loss is 0.00359838, the validation accuracy is 0.9802\n",
      "after 16300 training steps, the loss is 0.000751301, the validation accuracy is 0.9808\n",
      "after 16400 training steps, the loss is 0.00161301, the validation accuracy is 0.9806\n",
      "after 16500 training steps, the loss is 0.00106755, the validation accuracy is 0.9806\n",
      "after 16600 training steps, the loss is 0.00158052, the validation accuracy is 0.9806\n",
      "after 16700 training steps, the loss is 0.00162423, the validation accuracy is 0.9806\n",
      "after 16800 training steps, the loss is 0.0010551, the validation accuracy is 0.9804\n",
      "after 16900 training steps, the loss is 0.00160779, the validation accuracy is 0.981\n",
      "after 17000 training steps, the loss is 0.000671864, the validation accuracy is 0.9802\n",
      "after 17100 training steps, the loss is 0.00195176, the validation accuracy is 0.9806\n",
      "after 17200 training steps, the loss is 0.00332083, the validation accuracy is 0.9808\n",
      "after 17300 training steps, the loss is 0.00150207, the validation accuracy is 0.9812\n",
      "after 17400 training steps, the loss is 0.00363238, the validation accuracy is 0.9806\n",
      "after 17500 training steps, the loss is 0.00202464, the validation accuracy is 0.9806\n",
      "after 17600 training steps, the loss is 0.00169208, the validation accuracy is 0.9808\n",
      "after 17700 training steps, the loss is 0.00197799, the validation accuracy is 0.9806\n",
      "after 17800 training steps, the loss is 0.00145745, the validation accuracy is 0.9808\n",
      "after 17900 training steps, the loss is 0.00179254, the validation accuracy is 0.981\n",
      "after 18000 training steps, the loss is 0.00222671, the validation accuracy is 0.981\n",
      "after 18100 training steps, the loss is 0.00170917, the validation accuracy is 0.981\n",
      "after 18200 training steps, the loss is 0.00120714, the validation accuracy is 0.9806\n",
      "after 18300 training steps, the loss is 0.00102751, the validation accuracy is 0.981\n",
      "after 18400 training steps, the loss is 0.00203145, the validation accuracy is 0.9808\n",
      "after 18500 training steps, the loss is 0.00243795, the validation accuracy is 0.9808\n",
      "after 18600 training steps, the loss is 0.00184397, the validation accuracy is 0.9812\n",
      "after 18700 training steps, the loss is 0.00128799, the validation accuracy is 0.9806\n",
      "after 18800 training steps, the loss is 0.00216816, the validation accuracy is 0.9812\n",
      "after 18900 training steps, the loss is 0.00339793, the validation accuracy is 0.9806\n",
      "after 19000 training steps, the loss is 0.00310552, the validation accuracy is 0.981\n",
      "after 19100 training steps, the loss is 0.000416031, the validation accuracy is 0.9812\n",
      "after 19200 training steps, the loss is 0.00117325, the validation accuracy is 0.9806\n",
      "after 19300 training steps, the loss is 0.000819428, the validation accuracy is 0.981\n",
      "after 19400 training steps, the loss is 0.00139454, the validation accuracy is 0.9812\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 19500 training steps, the loss is 0.00130205, the validation accuracy is 0.9812\n",
      "after 19600 training steps, the loss is 0.00226193, the validation accuracy is 0.981\n",
      "after 19700 training steps, the loss is 0.00051925, the validation accuracy is 0.9808\n",
      "after 19800 training steps, the loss is 0.00116753, the validation accuracy is 0.9814\n",
      "after 19900 training steps, the loss is 0.00142629, the validation accuracy is 0.9812\n",
      "the training is finish!\n",
      "the test accuarcy is: 0.978\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "        #定义验证集与测试集\n",
    "    validate_data = {\n",
    "        x: mnist.validation.images,\n",
    "        y: mnist.validation.labels,\n",
    "    }\n",
    "    test_data = {x: mnist.test.images, y: mnist.test.labels}\n",
    "\n",
    "    for i in range(trainig_step):\n",
    "        xs, ys = mnist.train.next_batch(batch_size)\n",
    "        _, loss = sess.run(\n",
    "            [optimizer, cross_entropy_loss],\n",
    "            feed_dict={\n",
    "                x: xs,\n",
    "                y: ys,\n",
    "                learning_rate: 0.5\n",
    "            })\n",
    "\n",
    "        #每100次训练打印一次损失值与验证准确率\n",
    "        if i > 0 and i % 100 == 0:\n",
    "            validate_accuracy = sess.run(accuracy, feed_dict=validate_data)\n",
    "            print(\n",
    "                \"after %d training steps, the loss is %g, the validation accuracy is %g\"\n",
    "                % (i, loss, validate_accuracy))\n",
    "            saver.save(sess, './model.ckpt', global_step=i)\n",
    "\n",
    "    print(\"the training is finish!\")\n",
    "    #最终的测试准确率\n",
    "    acc = sess.run(accuracy, feed_dict=test_data)\n",
    "    print(\"the test accuarcy is:\", acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "可以看到增加训练轮数能够提高测试集的成绩，能够提高训练集的效果，但却无法提高校验集的效果"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "接着重新构建神经网络，搭建一个隐层，增加每层神经元个数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From D:\\Users\\Administrator\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n"
     ]
    }
   ],
   "source": [
    "L1_units_count = 300 #第一层神经元的个数\n",
    "W_1 = tf.Variable(initialize([784, L1_units_count])) #variable主要用于数据存储,在计算图的运算过程中，其值会一直保存到程序运行结束\n",
    "b_1 = tf.Variable(initialize([L1_units_count]))#比如神经网络中的权重和bias等，在训练过后，总是希望这些参数能够保存下来，而不是直接就消失了，所以这个时候要用到Variable\n",
    "logits_1 = tf.matmul(x, W_1) + b_1 #logits函数，即y=wx+b在tensorflow中计算图的定义方法\n",
    "output_1 = tf.nn.relu(logits_1) #激活函数为relu\n",
    "L2_units_count = 100 #增加隐层\n",
    "W_2 = tf.Variable(initialize([L1_units_count, L2_units_count]))\n",
    "b_2 = tf.Variable(initialize([L2_units_count]))\n",
    "logits_2 = tf.matmul(output_1, W_2) + b_2  \n",
    "output_2 = tf.nn.relu(logits_2) #激活函数为relu\n",
    "L3_units_count = 10 #输出层\n",
    "W_3 = tf.Variable(initialize([L2_units_count, L3_units_count]))\n",
    "b_3 = tf.Variable(initialize([L3_units_count]))\n",
    "logits_3 = tf.matmul(output_2, W_3) + b_3\n",
    "logits = logits_3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "其他不变"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 100 training steps, the loss is 0.371323, the validation accuracy is 0.8968\n",
      "after 200 training steps, the loss is 0.260083, the validation accuracy is 0.937\n",
      "after 300 training steps, the loss is 0.224699, the validation accuracy is 0.9346\n",
      "after 400 training steps, the loss is 0.252566, the validation accuracy is 0.9364\n",
      "after 500 training steps, the loss is 0.281817, the validation accuracy is 0.9552\n",
      "after 600 training steps, the loss is 0.0874335, the validation accuracy is 0.9546\n",
      "WARNING:tensorflow:From D:\\Users\\Administrator\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\training\\saver.py:966: remove_checkpoint (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to delete files with this prefix.\n",
      "after 700 training steps, the loss is 0.0852279, the validation accuracy is 0.9652\n",
      "after 800 training steps, the loss is 0.223299, the validation accuracy is 0.9606\n",
      "after 900 training steps, the loss is 0.0986793, the validation accuracy is 0.9676\n",
      "after 1000 training steps, the loss is 0.0666956, the validation accuracy is 0.9698\n",
      "after 1100 training steps, the loss is 0.0876692, the validation accuracy is 0.97\n",
      "after 1200 training steps, the loss is 0.041982, the validation accuracy is 0.9688\n",
      "after 1300 training steps, the loss is 0.0725941, the validation accuracy is 0.9734\n",
      "after 1400 training steps, the loss is 0.0488599, the validation accuracy is 0.9732\n",
      "after 1500 training steps, the loss is 0.0267214, the validation accuracy is 0.9752\n",
      "after 1600 training steps, the loss is 0.152448, the validation accuracy is 0.9748\n",
      "after 1700 training steps, the loss is 0.0654933, the validation accuracy is 0.973\n",
      "after 1800 training steps, the loss is 0.0221399, the validation accuracy is 0.9756\n",
      "after 1900 training steps, the loss is 0.0757319, the validation accuracy is 0.9724\n",
      "after 2000 training steps, the loss is 0.0961954, the validation accuracy is 0.9764\n",
      "after 2100 training steps, the loss is 0.0611784, the validation accuracy is 0.9752\n",
      "after 2200 training steps, the loss is 0.0564183, the validation accuracy is 0.975\n",
      "after 2300 training steps, the loss is 0.0483371, the validation accuracy is 0.9722\n",
      "after 2400 training steps, the loss is 0.0616464, the validation accuracy is 0.9766\n",
      "after 2500 training steps, the loss is 0.0465277, the validation accuracy is 0.9774\n",
      "after 2600 training steps, the loss is 0.0185933, the validation accuracy is 0.9746\n",
      "after 2700 training steps, the loss is 0.0579996, the validation accuracy is 0.9776\n",
      "after 2800 training steps, the loss is 0.069815, the validation accuracy is 0.9798\n",
      "after 2900 training steps, the loss is 0.0206211, the validation accuracy is 0.979\n",
      "after 3000 training steps, the loss is 0.0860407, the validation accuracy is 0.9782\n",
      "after 3100 training steps, the loss is 0.0596955, the validation accuracy is 0.9774\n",
      "after 3200 training steps, the loss is 0.0322254, the validation accuracy is 0.9764\n",
      "after 3300 training steps, the loss is 0.0146003, the validation accuracy is 0.9804\n",
      "after 3400 training steps, the loss is 0.00898865, the validation accuracy is 0.9786\n",
      "after 3500 training steps, the loss is 0.0131177, the validation accuracy is 0.9788\n",
      "after 3600 training steps, the loss is 0.00411192, the validation accuracy is 0.9792\n",
      "after 3700 training steps, the loss is 0.0199874, the validation accuracy is 0.9806\n",
      "after 3800 training steps, the loss is 0.054219, the validation accuracy is 0.9786\n",
      "after 3900 training steps, the loss is 0.00918517, the validation accuracy is 0.981\n",
      "after 4000 training steps, the loss is 0.00649741, the validation accuracy is 0.9808\n",
      "after 4100 training steps, the loss is 0.00663877, the validation accuracy is 0.9802\n",
      "after 4200 training steps, the loss is 0.0113162, the validation accuracy is 0.9796\n",
      "after 4300 training steps, the loss is 0.00788146, the validation accuracy is 0.9818\n",
      "after 4400 training steps, the loss is 0.017278, the validation accuracy is 0.981\n",
      "after 4500 training steps, the loss is 0.00675816, the validation accuracy is 0.9832\n",
      "after 4600 training steps, the loss is 0.0437443, the validation accuracy is 0.9804\n",
      "after 4700 training steps, the loss is 0.00640368, the validation accuracy is 0.9826\n",
      "after 4800 training steps, the loss is 0.0405882, the validation accuracy is 0.9782\n",
      "after 4900 training steps, the loss is 0.00452683, the validation accuracy is 0.9824\n",
      "after 5000 training steps, the loss is 0.00451917, the validation accuracy is 0.9818\n",
      "after 5100 training steps, the loss is 0.00502706, the validation accuracy is 0.9816\n",
      "after 5200 training steps, the loss is 0.0033282, the validation accuracy is 0.9824\n",
      "after 5300 training steps, the loss is 0.00565617, the validation accuracy is 0.9828\n",
      "after 5400 training steps, the loss is 0.00202403, the validation accuracy is 0.9824\n",
      "after 5500 training steps, the loss is 0.00311222, the validation accuracy is 0.981\n",
      "after 5600 training steps, the loss is 0.00329443, the validation accuracy is 0.9838\n",
      "after 5700 training steps, the loss is 0.00180249, the validation accuracy is 0.983\n",
      "after 5800 training steps, the loss is 0.00201325, the validation accuracy is 0.9832\n",
      "after 5900 training steps, the loss is 0.00607146, the validation accuracy is 0.9824\n",
      "after 6000 training steps, the loss is 0.00987351, the validation accuracy is 0.9816\n",
      "after 6100 training steps, the loss is 0.0038082, the validation accuracy is 0.9836\n",
      "after 6200 training steps, the loss is 0.00294057, the validation accuracy is 0.9836\n",
      "after 6300 training steps, the loss is 0.00310499, the validation accuracy is 0.9832\n",
      "after 6400 training steps, the loss is 0.0032192, the validation accuracy is 0.983\n",
      "after 6500 training steps, the loss is 0.00148359, the validation accuracy is 0.9828\n",
      "after 6600 training steps, the loss is 0.0048796, the validation accuracy is 0.9822\n",
      "after 6700 training steps, the loss is 0.00119408, the validation accuracy is 0.9832\n",
      "after 6800 training steps, the loss is 0.00224996, the validation accuracy is 0.983\n",
      "after 6900 training steps, the loss is 0.003298, the validation accuracy is 0.983\n",
      "after 7000 training steps, the loss is 0.0028267, the validation accuracy is 0.9826\n",
      "after 7100 training steps, the loss is 0.00137616, the validation accuracy is 0.983\n",
      "after 7200 training steps, the loss is 0.00141051, the validation accuracy is 0.983\n",
      "after 7300 training steps, the loss is 0.00155098, the validation accuracy is 0.9828\n",
      "after 7400 training steps, the loss is 0.00168784, the validation accuracy is 0.9832\n",
      "after 7500 training steps, the loss is 0.00342533, the validation accuracy is 0.9832\n",
      "after 7600 training steps, the loss is 0.00182903, the validation accuracy is 0.983\n",
      "after 7700 training steps, the loss is 0.000892377, the validation accuracy is 0.982\n",
      "after 7800 training steps, the loss is 0.00300193, the validation accuracy is 0.9828\n",
      "after 7900 training steps, the loss is 0.000636118, the validation accuracy is 0.9828\n",
      "after 8000 training steps, the loss is 0.00065063, the validation accuracy is 0.9832\n",
      "after 8100 training steps, the loss is 0.00144627, the validation accuracy is 0.983\n",
      "after 8200 training steps, the loss is 0.00192463, the validation accuracy is 0.9832\n",
      "after 8300 training steps, the loss is 0.00120452, the validation accuracy is 0.983\n",
      "after 8400 training steps, the loss is 0.001082, the validation accuracy is 0.9838\n",
      "after 8500 training steps, the loss is 0.00239121, the validation accuracy is 0.9832\n",
      "after 8600 training steps, the loss is 0.000399114, the validation accuracy is 0.9828\n",
      "after 8700 training steps, the loss is 0.000644488, the validation accuracy is 0.9832\n",
      "after 8800 training steps, the loss is 0.00127659, the validation accuracy is 0.9832\n",
      "after 8900 training steps, the loss is 0.000885845, the validation accuracy is 0.983\n",
      "after 9000 training steps, the loss is 0.00244388, the validation accuracy is 0.9826\n",
      "after 9100 training steps, the loss is 0.00147469, the validation accuracy is 0.9828\n",
      "after 9200 training steps, the loss is 0.000692757, the validation accuracy is 0.983\n",
      "after 9300 training steps, the loss is 0.00047271, the validation accuracy is 0.983\n",
      "after 9400 training steps, the loss is 0.000894911, the validation accuracy is 0.9824\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 9500 training steps, the loss is 0.000351363, the validation accuracy is 0.9836\n",
      "after 9600 training steps, the loss is 0.000688665, the validation accuracy is 0.983\n",
      "after 9700 training steps, the loss is 0.000937123, the validation accuracy is 0.9828\n",
      "after 9800 training steps, the loss is 0.000341652, the validation accuracy is 0.9828\n",
      "after 9900 training steps, the loss is 0.00097784, the validation accuracy is 0.9826\n",
      "after 10000 training steps, the loss is 0.000573671, the validation accuracy is 0.9832\n",
      "after 10100 training steps, the loss is 0.00067316, the validation accuracy is 0.9834\n",
      "after 10200 training steps, the loss is 0.000443998, the validation accuracy is 0.9832\n",
      "after 10300 training steps, the loss is 0.000210874, the validation accuracy is 0.9832\n",
      "after 10400 training steps, the loss is 0.000690768, the validation accuracy is 0.9834\n",
      "after 10500 training steps, the loss is 0.0013713, the validation accuracy is 0.9836\n",
      "after 10600 training steps, the loss is 0.000785943, the validation accuracy is 0.9832\n",
      "after 10700 training steps, the loss is 0.000239446, the validation accuracy is 0.9834\n",
      "after 10800 training steps, the loss is 0.000502698, the validation accuracy is 0.9828\n",
      "after 10900 training steps, the loss is 0.000458775, the validation accuracy is 0.9822\n",
      "after 11000 training steps, the loss is 0.000901364, the validation accuracy is 0.9834\n",
      "after 11100 training steps, the loss is 0.00067571, the validation accuracy is 0.983\n",
      "after 11200 training steps, the loss is 0.000794554, the validation accuracy is 0.983\n",
      "after 11300 training steps, the loss is 0.00149759, the validation accuracy is 0.983\n",
      "after 11400 training steps, the loss is 0.00029485, the validation accuracy is 0.9838\n",
      "after 11500 training steps, the loss is 0.00019876, the validation accuracy is 0.9836\n",
      "after 11600 training steps, the loss is 0.00158049, the validation accuracy is 0.9836\n",
      "after 11700 training steps, the loss is 0.000461965, the validation accuracy is 0.9836\n",
      "after 11800 training steps, the loss is 0.000640394, the validation accuracy is 0.9836\n",
      "after 11900 training steps, the loss is 0.000895348, the validation accuracy is 0.9838\n",
      "after 12000 training steps, the loss is 0.000768735, the validation accuracy is 0.9834\n",
      "after 12100 training steps, the loss is 0.000313295, the validation accuracy is 0.9834\n",
      "after 12200 training steps, the loss is 0.000428011, the validation accuracy is 0.9832\n",
      "after 12300 training steps, the loss is 0.000423069, the validation accuracy is 0.9834\n",
      "after 12400 training steps, the loss is 0.000580985, the validation accuracy is 0.984\n",
      "after 12500 training steps, the loss is 0.000596969, the validation accuracy is 0.9832\n",
      "after 12600 training steps, the loss is 0.000442073, the validation accuracy is 0.9836\n",
      "after 12700 training steps, the loss is 0.000268454, the validation accuracy is 0.9838\n",
      "after 12800 training steps, the loss is 0.000206785, the validation accuracy is 0.9836\n",
      "after 12900 training steps, the loss is 0.000778469, the validation accuracy is 0.9836\n",
      "after 13000 training steps, the loss is 0.000311607, the validation accuracy is 0.9836\n",
      "after 13100 training steps, the loss is 0.000270882, the validation accuracy is 0.9832\n",
      "after 13200 training steps, the loss is 7.39575e-05, the validation accuracy is 0.9838\n",
      "after 13300 training steps, the loss is 0.000181735, the validation accuracy is 0.9836\n",
      "after 13400 training steps, the loss is 0.000528278, the validation accuracy is 0.9836\n",
      "after 13500 training steps, the loss is 0.000901079, the validation accuracy is 0.9834\n",
      "after 13600 training steps, the loss is 0.000723661, the validation accuracy is 0.9834\n",
      "after 13700 training steps, the loss is 0.00088276, the validation accuracy is 0.9836\n",
      "after 13800 training steps, the loss is 0.00022705, the validation accuracy is 0.9836\n",
      "after 13900 training steps, the loss is 0.000330551, the validation accuracy is 0.9838\n",
      "after 14000 training steps, the loss is 0.000330738, the validation accuracy is 0.9838\n",
      "after 14100 training steps, the loss is 0.000132957, the validation accuracy is 0.9836\n",
      "after 14200 training steps, the loss is 0.000250562, the validation accuracy is 0.9834\n",
      "after 14300 training steps, the loss is 0.000444232, the validation accuracy is 0.9834\n",
      "after 14400 training steps, the loss is 0.000423463, the validation accuracy is 0.9836\n",
      "after 14500 training steps, the loss is 0.000304913, the validation accuracy is 0.9834\n",
      "after 14600 training steps, the loss is 0.000431029, the validation accuracy is 0.9832\n",
      "after 14700 training steps, the loss is 0.00037798, the validation accuracy is 0.9834\n",
      "after 14800 training steps, the loss is 7.8527e-05, the validation accuracy is 0.9836\n",
      "after 14900 training steps, the loss is 0.000294468, the validation accuracy is 0.9836\n",
      "after 15000 training steps, the loss is 0.000669251, the validation accuracy is 0.9834\n",
      "after 15100 training steps, the loss is 0.000487306, the validation accuracy is 0.9832\n",
      "after 15200 training steps, the loss is 0.000290202, the validation accuracy is 0.9836\n",
      "after 15300 training steps, the loss is 0.00040975, the validation accuracy is 0.9832\n",
      "after 15400 training steps, the loss is 0.000315965, the validation accuracy is 0.9834\n",
      "after 15500 training steps, the loss is 0.000238473, the validation accuracy is 0.9838\n",
      "after 15600 training steps, the loss is 0.000616899, the validation accuracy is 0.9834\n",
      "after 15700 training steps, the loss is 0.000458564, the validation accuracy is 0.9834\n",
      "after 15800 training steps, the loss is 0.000307879, the validation accuracy is 0.9834\n",
      "after 15900 training steps, the loss is 0.000908637, the validation accuracy is 0.9834\n",
      "after 16000 training steps, the loss is 0.000290175, the validation accuracy is 0.9834\n",
      "after 16100 training steps, the loss is 0.000402773, the validation accuracy is 0.9838\n",
      "after 16200 training steps, the loss is 0.000471494, the validation accuracy is 0.9836\n",
      "after 16300 training steps, the loss is 0.000227053, the validation accuracy is 0.9834\n",
      "after 16400 training steps, the loss is 0.000971544, the validation accuracy is 0.9834\n",
      "after 16500 training steps, the loss is 0.000390822, the validation accuracy is 0.9836\n",
      "after 16600 training steps, the loss is 0.00030112, the validation accuracy is 0.9834\n",
      "after 16700 training steps, the loss is 0.000369529, the validation accuracy is 0.9836\n",
      "after 16800 training steps, the loss is 0.000197763, the validation accuracy is 0.9838\n",
      "after 16900 training steps, the loss is 0.000370483, the validation accuracy is 0.9836\n",
      "after 17000 training steps, the loss is 0.000388771, the validation accuracy is 0.9838\n",
      "after 17100 training steps, the loss is 0.000773371, the validation accuracy is 0.9834\n",
      "after 17200 training steps, the loss is 0.00104009, the validation accuracy is 0.9836\n",
      "after 17300 training steps, the loss is 0.00011835, the validation accuracy is 0.9834\n",
      "after 17400 training steps, the loss is 0.00010433, the validation accuracy is 0.9836\n",
      "after 17500 training steps, the loss is 0.000300441, the validation accuracy is 0.9834\n",
      "after 17600 training steps, the loss is 0.000558232, the validation accuracy is 0.9836\n",
      "after 17700 training steps, the loss is 0.000675834, the validation accuracy is 0.9836\n",
      "after 17800 training steps, the loss is 0.000334623, the validation accuracy is 0.9834\n",
      "after 17900 training steps, the loss is 0.000203965, the validation accuracy is 0.9838\n",
      "after 18000 training steps, the loss is 0.000436742, the validation accuracy is 0.9834\n",
      "after 18100 training steps, the loss is 0.000370016, the validation accuracy is 0.9836\n",
      "after 18200 training steps, the loss is 8.27657e-05, the validation accuracy is 0.9834\n",
      "after 18300 training steps, the loss is 0.000184885, the validation accuracy is 0.9836\n",
      "after 18400 training steps, the loss is 0.000306513, the validation accuracy is 0.9834\n",
      "after 18500 training steps, the loss is 0.000191553, the validation accuracy is 0.9836\n",
      "after 18600 training steps, the loss is 0.00036796, the validation accuracy is 0.9838\n",
      "after 18700 training steps, the loss is 0.000505167, the validation accuracy is 0.9838\n",
      "after 18800 training steps, the loss is 0.000188453, the validation accuracy is 0.9836\n",
      "after 18900 training steps, the loss is 0.000142586, the validation accuracy is 0.9836\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 19000 training steps, the loss is 0.000407957, the validation accuracy is 0.9834\n",
      "after 19100 training steps, the loss is 0.000242787, the validation accuracy is 0.9834\n",
      "after 19200 training steps, the loss is 0.000159002, the validation accuracy is 0.9834\n",
      "after 19300 training steps, the loss is 0.000381858, the validation accuracy is 0.9838\n",
      "after 19400 training steps, the loss is 0.000264684, the validation accuracy is 0.9836\n",
      "after 19500 training steps, the loss is 0.000331532, the validation accuracy is 0.9836\n",
      "after 19600 training steps, the loss is 8.09229e-05, the validation accuracy is 0.9836\n",
      "after 19700 training steps, the loss is 0.000312704, the validation accuracy is 0.9836\n",
      "after 19800 training steps, the loss is 0.000161217, the validation accuracy is 0.9834\n",
      "after 19900 training steps, the loss is 0.000420922, the validation accuracy is 0.9836\n",
      "the training is finish!\n",
      "the test accuarcy is: 0.9819\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "        #定义验证集与测试集\n",
    "    validate_data = {\n",
    "        x: mnist.validation.images,\n",
    "        y: mnist.validation.labels,\n",
    "    }\n",
    "    test_data = {x: mnist.test.images, y: mnist.test.labels}\n",
    "\n",
    "    for i in range(trainig_step):\n",
    "        xs, ys = mnist.train.next_batch(batch_size)\n",
    "        _, loss = sess.run(\n",
    "            [optimizer, cross_entropy_loss],\n",
    "            feed_dict={\n",
    "                x: xs,\n",
    "                y: ys,\n",
    "                learning_rate: 0.3\n",
    "            })\n",
    "\n",
    "        #每100次训练打印一次损失值与验证准确率\n",
    "        if i > 0 and i % 100 == 0:\n",
    "            validate_accuracy = sess.run(accuracy, feed_dict=validate_data)\n",
    "            print(\n",
    "                \"after %d training steps, the loss is %g, the validation accuracy is %g\"\n",
    "                % (i, loss, validate_accuracy))\n",
    "            saver.save(sess, './model.ckpt', global_step=i)\n",
    "\n",
    "    print(\"the training is finish!\")\n",
    "    #最终的测试准确率\n",
    "    acc = sess.run(accuracy, feed_dict=test_data)\n",
    "    print(\"the test accuarcy is:\", acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "增加一层隐层之后可以发现收敛明显变迅速了，效果也变好了。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "接下来增加神经元的个数试试看"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "L1_units_count = 500 #第一层神经元的个数\n",
    "W_1 = tf.Variable(initialize([784, L1_units_count])) #variable主要用于数据存储,在计算图的运算过程中，其值会一直保存到程序运行结束\n",
    "b_1 = tf.Variable(initialize([L1_units_count]))#比如神经网络中的权重和bias等，在训练过后，总是希望这些参数能够保存下来，而不是直接就消失了，所以这个时候要用到Variable\n",
    "logits_1 = tf.matmul(x, W_1) + b_1 #logits函数，即y=wx+b在tensorflow中计算图的定义方法\n",
    "output_1 = tf.nn.relu(logits_1) #激活函数为relu\n",
    "L2_units_count =300 #增加隐层\n",
    "W_2 = tf.Variable(initialize([L1_units_count, L2_units_count]))\n",
    "b_2 = tf.Variable(initialize([L2_units_count]))\n",
    "logits_2 = tf.matmul(output_1, W_2) + b_2  \n",
    "output_2 = tf.nn.relu(logits_2) #激活函数为relu\n",
    "L3_units_count = 10 #输出层\n",
    "W_3 = tf.Variable(initialize([L2_units_count, L3_units_count]))\n",
    "b_3 = tf.Variable(initialize([L3_units_count]))\n",
    "logits_3 = tf.matmul(output_2, W_3) + b_3\n",
    "logits = logits_3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 100 training steps, the loss is 0.544026, the validation accuracy is 0.8976\n",
      "after 200 training steps, the loss is 0.41729, the validation accuracy is 0.922\n",
      "after 300 training steps, the loss is 0.153633, the validation accuracy is 0.9424\n",
      "after 400 training steps, the loss is 0.0894054, the validation accuracy is 0.9564\n",
      "after 500 training steps, the loss is 0.126155, the validation accuracy is 0.96\n",
      "after 600 training steps, the loss is 0.0975901, the validation accuracy is 0.9634\n",
      "after 700 training steps, the loss is 0.119527, the validation accuracy is 0.9668\n",
      "after 800 training steps, the loss is 0.196949, the validation accuracy is 0.9618\n",
      "after 900 training steps, the loss is 0.0831493, the validation accuracy is 0.9666\n",
      "after 1000 training steps, the loss is 0.0292791, the validation accuracy is 0.9708\n",
      "after 1100 training steps, the loss is 0.0535944, the validation accuracy is 0.9722\n",
      "after 1200 training steps, the loss is 0.0622088, the validation accuracy is 0.9694\n",
      "after 1300 training steps, the loss is 0.059521, the validation accuracy is 0.9754\n",
      "after 1400 training steps, the loss is 0.114228, the validation accuracy is 0.975\n",
      "after 1500 training steps, the loss is 0.0614584, the validation accuracy is 0.9772\n",
      "after 1600 training steps, the loss is 0.00657673, the validation accuracy is 0.974\n",
      "after 1700 training steps, the loss is 0.0665577, the validation accuracy is 0.977\n",
      "after 1800 training steps, the loss is 0.0445739, the validation accuracy is 0.9764\n",
      "after 1900 training steps, the loss is 0.0779101, the validation accuracy is 0.9724\n",
      "after 2000 training steps, the loss is 0.0417483, the validation accuracy is 0.978\n",
      "after 2100 training steps, the loss is 0.0889887, the validation accuracy is 0.9788\n",
      "after 2200 training steps, the loss is 0.0221923, the validation accuracy is 0.9788\n",
      "after 2300 training steps, the loss is 0.0569037, the validation accuracy is 0.9792\n",
      "after 2400 training steps, the loss is 0.0207774, the validation accuracy is 0.9796\n",
      "after 2500 training steps, the loss is 0.0121986, the validation accuracy is 0.9784\n",
      "after 2600 training steps, the loss is 0.0329468, the validation accuracy is 0.9796\n",
      "after 2700 training steps, the loss is 0.0083059, the validation accuracy is 0.9738\n",
      "after 2800 training steps, the loss is 0.0359777, the validation accuracy is 0.9786\n",
      "after 2900 training steps, the loss is 0.00729707, the validation accuracy is 0.98\n",
      "after 3000 training steps, the loss is 0.0289334, the validation accuracy is 0.9796\n",
      "after 3100 training steps, the loss is 0.031564, the validation accuracy is 0.9794\n",
      "after 3200 training steps, the loss is 0.0116107, the validation accuracy is 0.9794\n",
      "after 3300 training steps, the loss is 0.0161712, the validation accuracy is 0.9822\n",
      "after 3400 training steps, the loss is 0.0214939, the validation accuracy is 0.9816\n",
      "after 3500 training steps, the loss is 0.130804, the validation accuracy is 0.978\n",
      "after 3600 training steps, the loss is 0.00583696, the validation accuracy is 0.976\n",
      "after 3700 training steps, the loss is 0.00705404, the validation accuracy is 0.9808\n",
      "after 3800 training steps, the loss is 0.0146168, the validation accuracy is 0.9828\n",
      "after 3900 training steps, the loss is 0.00509052, the validation accuracy is 0.9828\n",
      "after 4000 training steps, the loss is 0.0181998, the validation accuracy is 0.98\n",
      "after 4100 training steps, the loss is 0.047132, the validation accuracy is 0.9778\n",
      "after 4200 training steps, the loss is 0.00228193, the validation accuracy is 0.9802\n",
      "after 4300 training steps, the loss is 0.00453897, the validation accuracy is 0.9826\n",
      "after 4400 training steps, the loss is 0.0356555, the validation accuracy is 0.9778\n",
      "after 4500 training steps, the loss is 0.0179379, the validation accuracy is 0.9838\n",
      "after 4600 training steps, the loss is 0.0107963, the validation accuracy is 0.9834\n",
      "after 4700 training steps, the loss is 0.111092, the validation accuracy is 0.9774\n",
      "after 4800 training steps, the loss is 0.00992277, the validation accuracy is 0.9812\n",
      "after 4900 training steps, the loss is 0.00104831, the validation accuracy is 0.9834\n",
      "after 5000 training steps, the loss is 0.00301395, the validation accuracy is 0.9832\n",
      "after 5100 training steps, the loss is 0.0027705, the validation accuracy is 0.9822\n",
      "after 5200 training steps, the loss is 0.00692052, the validation accuracy is 0.9828\n",
      "after 5300 training steps, the loss is 0.019645, the validation accuracy is 0.9804\n",
      "after 5400 training steps, the loss is 0.00496559, the validation accuracy is 0.9836\n",
      "after 5500 training steps, the loss is 0.00351142, the validation accuracy is 0.9832\n",
      "after 5600 training steps, the loss is 0.00200093, the validation accuracy is 0.982\n",
      "after 5700 training steps, the loss is 0.0154127, the validation accuracy is 0.9824\n",
      "after 5800 training steps, the loss is 0.00772928, the validation accuracy is 0.9832\n",
      "after 5900 training steps, the loss is 0.00294627, the validation accuracy is 0.9834\n",
      "after 6000 training steps, the loss is 0.00155515, the validation accuracy is 0.9838\n",
      "after 6100 training steps, the loss is 0.00101993, the validation accuracy is 0.9834\n",
      "after 6200 training steps, the loss is 0.00229669, the validation accuracy is 0.9834\n",
      "after 6300 training steps, the loss is 0.00225666, the validation accuracy is 0.9828\n",
      "after 6400 training steps, the loss is 0.000795889, the validation accuracy is 0.9826\n",
      "after 6500 training steps, the loss is 0.00217911, the validation accuracy is 0.9836\n",
      "after 6600 training steps, the loss is 0.00248104, the validation accuracy is 0.9836\n",
      "after 6700 training steps, the loss is 0.00212882, the validation accuracy is 0.9832\n",
      "after 6800 training steps, the loss is 0.00232992, the validation accuracy is 0.9838\n",
      "after 6900 training steps, the loss is 0.00272286, the validation accuracy is 0.984\n",
      "after 7000 training steps, the loss is 0.000903103, the validation accuracy is 0.9826\n",
      "after 7100 training steps, the loss is 0.00175096, the validation accuracy is 0.9836\n",
      "after 7200 training steps, the loss is 0.00158093, the validation accuracy is 0.9838\n",
      "after 7300 training steps, the loss is 0.00294484, the validation accuracy is 0.9838\n",
      "after 7400 training steps, the loss is 0.00185256, the validation accuracy is 0.9838\n",
      "after 7500 training steps, the loss is 0.000642804, the validation accuracy is 0.9836\n",
      "after 7600 training steps, the loss is 0.00209774, the validation accuracy is 0.9838\n",
      "after 7700 training steps, the loss is 0.00242372, the validation accuracy is 0.9834\n",
      "after 7800 training steps, the loss is 0.000688595, the validation accuracy is 0.9828\n",
      "after 7900 training steps, the loss is 0.000638051, the validation accuracy is 0.9832\n",
      "after 8000 training steps, the loss is 0.00105903, the validation accuracy is 0.9838\n",
      "after 8100 training steps, the loss is 0.000687546, the validation accuracy is 0.9842\n",
      "after 8200 training steps, the loss is 0.000419807, the validation accuracy is 0.9836\n",
      "after 8300 training steps, the loss is 0.000634119, the validation accuracy is 0.9842\n",
      "after 8400 training steps, the loss is 0.00142278, the validation accuracy is 0.9834\n",
      "after 8500 training steps, the loss is 0.00133769, the validation accuracy is 0.984\n",
      "after 8600 training steps, the loss is 0.00139319, the validation accuracy is 0.9834\n",
      "after 8700 training steps, the loss is 0.00113644, the validation accuracy is 0.983\n",
      "after 8800 training steps, the loss is 0.000469966, the validation accuracy is 0.9842\n",
      "after 8900 training steps, the loss is 0.00118902, the validation accuracy is 0.984\n",
      "after 9000 training steps, the loss is 0.00084915, the validation accuracy is 0.9836\n",
      "after 9100 training steps, the loss is 0.00117434, the validation accuracy is 0.984\n",
      "after 9200 training steps, the loss is 0.00103809, the validation accuracy is 0.9838\n",
      "after 9300 training steps, the loss is 0.000491512, the validation accuracy is 0.9838\n",
      "after 9400 training steps, the loss is 0.00113122, the validation accuracy is 0.9838\n",
      "after 9500 training steps, the loss is 0.000748395, the validation accuracy is 0.9836\n",
      "after 9600 training steps, the loss is 0.000886577, the validation accuracy is 0.9842\n",
      "after 9700 training steps, the loss is 0.000378048, the validation accuracy is 0.9838\n",
      "after 9800 training steps, the loss is 0.000379916, the validation accuracy is 0.9842\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 9900 training steps, the loss is 0.000895743, the validation accuracy is 0.983\n",
      "after 10000 training steps, the loss is 0.000683011, the validation accuracy is 0.984\n",
      "after 10100 training steps, the loss is 0.000372479, the validation accuracy is 0.9844\n",
      "after 10200 training steps, the loss is 0.000546577, the validation accuracy is 0.984\n",
      "after 10300 training steps, the loss is 0.000754764, the validation accuracy is 0.9836\n",
      "after 10400 training steps, the loss is 0.000760825, the validation accuracy is 0.9834\n",
      "after 10500 training steps, the loss is 0.000682304, the validation accuracy is 0.9838\n",
      "after 10600 training steps, the loss is 0.000361395, the validation accuracy is 0.9834\n",
      "after 10700 training steps, the loss is 0.000423711, the validation accuracy is 0.9834\n",
      "after 10800 training steps, the loss is 0.00082536, the validation accuracy is 0.9838\n",
      "after 10900 training steps, the loss is 0.00097472, the validation accuracy is 0.9838\n",
      "after 11000 training steps, the loss is 0.000891151, the validation accuracy is 0.9836\n",
      "after 11100 training steps, the loss is 0.00099152, the validation accuracy is 0.9836\n",
      "after 11200 training steps, the loss is 0.00021658, the validation accuracy is 0.9836\n",
      "after 11300 training steps, the loss is 0.000233404, the validation accuracy is 0.9838\n",
      "after 11400 training steps, the loss is 0.000472848, the validation accuracy is 0.9836\n",
      "after 11500 training steps, the loss is 0.000892963, the validation accuracy is 0.9834\n",
      "after 11600 training steps, the loss is 0.000687672, the validation accuracy is 0.9838\n",
      "after 11700 training steps, the loss is 0.000746735, the validation accuracy is 0.9832\n",
      "after 11800 training steps, the loss is 0.000693611, the validation accuracy is 0.9838\n",
      "after 11900 training steps, the loss is 0.00024844, the validation accuracy is 0.9836\n",
      "after 12000 training steps, the loss is 0.000316072, the validation accuracy is 0.9836\n",
      "after 12100 training steps, the loss is 0.000343081, the validation accuracy is 0.984\n",
      "after 12200 training steps, the loss is 0.000440377, the validation accuracy is 0.9836\n",
      "after 12300 training steps, the loss is 0.000769713, the validation accuracy is 0.9836\n",
      "after 12400 training steps, the loss is 0.000530156, the validation accuracy is 0.9836\n",
      "after 12500 training steps, the loss is 0.000253536, the validation accuracy is 0.9842\n",
      "after 12600 training steps, the loss is 0.000562274, the validation accuracy is 0.9836\n",
      "after 12700 training steps, the loss is 0.000464626, the validation accuracy is 0.9842\n",
      "after 12800 training steps, the loss is 0.000522151, the validation accuracy is 0.9834\n",
      "after 12900 training steps, the loss is 0.000640764, the validation accuracy is 0.9842\n",
      "after 13000 training steps, the loss is 0.000188732, the validation accuracy is 0.9836\n",
      "after 13100 training steps, the loss is 0.000205575, the validation accuracy is 0.9836\n",
      "after 13200 training steps, the loss is 0.000237854, the validation accuracy is 0.9836\n",
      "after 13300 training steps, the loss is 0.000497287, the validation accuracy is 0.9838\n",
      "after 13400 training steps, the loss is 0.000162664, the validation accuracy is 0.984\n",
      "after 13500 training steps, the loss is 0.000810218, the validation accuracy is 0.9836\n",
      "after 13600 training steps, the loss is 0.0001578, the validation accuracy is 0.9834\n",
      "after 13700 training steps, the loss is 0.000688641, the validation accuracy is 0.984\n",
      "after 13800 training steps, the loss is 0.000294359, the validation accuracy is 0.984\n",
      "after 13900 training steps, the loss is 0.000590421, the validation accuracy is 0.9838\n",
      "after 14000 training steps, the loss is 0.000640135, the validation accuracy is 0.9836\n",
      "after 14100 training steps, the loss is 0.000386934, the validation accuracy is 0.9836\n",
      "after 14200 training steps, the loss is 0.000297442, the validation accuracy is 0.9838\n",
      "after 14300 training steps, the loss is 0.000476433, the validation accuracy is 0.984\n",
      "after 14400 training steps, the loss is 0.000569218, the validation accuracy is 0.9842\n",
      "after 14500 training steps, the loss is 0.000270691, the validation accuracy is 0.9836\n",
      "after 14600 training steps, the loss is 0.000435854, the validation accuracy is 0.9838\n",
      "after 14700 training steps, the loss is 0.000521073, the validation accuracy is 0.9834\n",
      "after 14800 training steps, the loss is 0.000852807, the validation accuracy is 0.9838\n",
      "after 14900 training steps, the loss is 0.000151026, the validation accuracy is 0.9836\n",
      "after 15000 training steps, the loss is 0.000205689, the validation accuracy is 0.9838\n",
      "after 15100 training steps, the loss is 0.000517087, the validation accuracy is 0.9836\n",
      "after 15200 training steps, the loss is 0.000181227, the validation accuracy is 0.9838\n",
      "after 15300 training steps, the loss is 0.000207655, the validation accuracy is 0.9838\n",
      "after 15400 training steps, the loss is 0.000189883, the validation accuracy is 0.9836\n",
      "after 15500 training steps, the loss is 0.000301225, the validation accuracy is 0.984\n",
      "after 15600 training steps, the loss is 0.000263957, the validation accuracy is 0.9836\n",
      "after 15700 training steps, the loss is 0.000711616, the validation accuracy is 0.9838\n",
      "after 15800 training steps, the loss is 0.000262386, the validation accuracy is 0.984\n",
      "after 15900 training steps, the loss is 0.000450785, the validation accuracy is 0.9836\n",
      "after 16000 training steps, the loss is 0.000529113, the validation accuracy is 0.9836\n",
      "after 16100 training steps, the loss is 0.000384772, the validation accuracy is 0.984\n",
      "after 16200 training steps, the loss is 0.000622788, the validation accuracy is 0.984\n",
      "after 16300 training steps, the loss is 0.000144723, the validation accuracy is 0.9838\n",
      "after 16400 training steps, the loss is 0.000244077, the validation accuracy is 0.984\n",
      "after 16500 training steps, the loss is 0.000409433, the validation accuracy is 0.9836\n",
      "after 16600 training steps, the loss is 0.000177238, the validation accuracy is 0.9838\n",
      "after 16700 training steps, the loss is 0.00069744, the validation accuracy is 0.984\n",
      "after 16800 training steps, the loss is 0.000289878, the validation accuracy is 0.9834\n",
      "after 16900 training steps, the loss is 0.00040625, the validation accuracy is 0.9834\n",
      "after 17000 training steps, the loss is 0.000169967, the validation accuracy is 0.9838\n",
      "after 17100 training steps, the loss is 0.000278461, the validation accuracy is 0.9838\n",
      "after 17200 training steps, the loss is 0.000215964, the validation accuracy is 0.984\n",
      "after 17300 training steps, the loss is 0.000441782, the validation accuracy is 0.9834\n",
      "after 17400 training steps, the loss is 0.000494748, the validation accuracy is 0.9838\n",
      "after 17500 training steps, the loss is 0.000414595, the validation accuracy is 0.9836\n",
      "after 17600 training steps, the loss is 0.000207965, the validation accuracy is 0.9836\n",
      "after 17700 training steps, the loss is 0.000301356, the validation accuracy is 0.984\n",
      "after 17800 training steps, the loss is 0.00023119, the validation accuracy is 0.984\n",
      "after 17900 training steps, the loss is 0.000186586, the validation accuracy is 0.9838\n",
      "after 18000 training steps, the loss is 0.000419061, the validation accuracy is 0.9836\n",
      "after 18100 training steps, the loss is 0.000198523, the validation accuracy is 0.9836\n",
      "after 18200 training steps, the loss is 0.000251711, the validation accuracy is 0.984\n",
      "after 18300 training steps, the loss is 0.000306423, the validation accuracy is 0.9838\n",
      "after 18400 training steps, the loss is 0.00018776, the validation accuracy is 0.9838\n",
      "after 18500 training steps, the loss is 0.00027787, the validation accuracy is 0.9838\n",
      "after 18600 training steps, the loss is 0.000133538, the validation accuracy is 0.9838\n",
      "after 18700 training steps, the loss is 0.000270057, the validation accuracy is 0.9834\n",
      "after 18800 training steps, the loss is 0.000597758, the validation accuracy is 0.9838\n",
      "after 18900 training steps, the loss is 0.000135975, the validation accuracy is 0.9838\n",
      "after 19000 training steps, the loss is 0.000378824, the validation accuracy is 0.9838\n",
      "after 19100 training steps, the loss is 0.00041612, the validation accuracy is 0.9838\n",
      "after 19200 training steps, the loss is 0.000424513, the validation accuracy is 0.984\n",
      "after 19300 training steps, the loss is 0.000349443, the validation accuracy is 0.9836\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 19400 training steps, the loss is 0.00029815, the validation accuracy is 0.9838\n",
      "after 19500 training steps, the loss is 0.000315337, the validation accuracy is 0.9836\n",
      "after 19600 training steps, the loss is 0.000173339, the validation accuracy is 0.984\n",
      "after 19700 training steps, the loss is 0.000273377, the validation accuracy is 0.9838\n",
      "after 19800 training steps, the loss is 0.000599989, the validation accuracy is 0.9838\n",
      "after 19900 training steps, the loss is 0.000125728, the validation accuracy is 0.9836\n",
      "the training is finish!\n",
      "the test accuarcy is: 0.9821\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "        #定义验证集与测试集\n",
    "    validate_data = {\n",
    "        x: mnist.validation.images,\n",
    "        y: mnist.validation.labels,\n",
    "    }\n",
    "    test_data = {x: mnist.test.images, y: mnist.test.labels}\n",
    "\n",
    "    for i in range(trainig_step):\n",
    "        xs, ys = mnist.train.next_batch(batch_size)\n",
    "        _, loss = sess.run(\n",
    "            [optimizer, cross_entropy_loss],\n",
    "            feed_dict={\n",
    "                x: xs,\n",
    "                y: ys,\n",
    "                learning_rate: 0.3\n",
    "            })\n",
    "\n",
    "        #每100次训练打印一次损失值与验证准确率\n",
    "        if i > 0 and i % 100 == 0:\n",
    "            validate_accuracy = sess.run(accuracy, feed_dict=validate_data)\n",
    "            print(\n",
    "                \"after %d training steps, the loss is %g, the validation accuracy is %g\"\n",
    "                % (i, loss, validate_accuracy))\n",
    "            saver.save(sess, './model.ckpt', global_step=i)\n",
    "\n",
    "    print(\"the training is finish!\")\n",
    "    #最终的测试准确率\n",
    "    acc = sess.run(accuracy, feed_dict=test_data)\n",
    "    print(\"the test accuarcy is:\", acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "可以看出，增加神经元数目收敛变快了，效果稍微好了一点点"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_weight(shape, lambda1):\n",
    "    var = tf.Variable(tf.truncated_normal(shape,stddev=0.1), dtype=tf.float32) # 生成一个变量\n",
    "    tf.add_to_collection('losses', tf.contrib.layers.l1_regularizer(lambda1)(var)) # add_to_collection()函数将新生成变量的L1正则化损失加入集合losses\n",
    "    return var"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "L1_units_count = 500 #第一层神经元的个数\n",
    "W_1 = get_weight([784, L1_units_count],0.001) #variable主要用于数据存储,在计算图的运算过程中，其值会一直保存到程序运行结束\n",
    "b_1 = tf.Variable(initialize([L1_units_count]))#比如神经网络中的权重和bias等，在训练过后，总是希望这些参数能够保存下来，而不是直接就消失了，所以这个时候要用到Variable\n",
    "logits_1 = tf.matmul(x, W_1) + b_1 #logits函数，即y=wx+b在tensorflow中计算图的定义方法\n",
    "output_1 = tf.nn.relu(logits_1) #激活函数为relu\n",
    "L2_units_count =300 #增加隐层\n",
    "W_2 = get_weight([L1_units_count, L2_units_count],0.001)\n",
    "b_2 = tf.Variable(initialize([L2_units_count]))\n",
    "logits_2 = tf.matmul(output_1, W_2) + b_2  \n",
    "output_2 = tf.nn.relu(logits_2) #激活函数为relu\n",
    "L3_units_count = 10 #输出层\n",
    "W_3 = get_weight([L2_units_count, L3_units_count],0.001)\n",
    "b_3 = tf.Variable(initialize([L3_units_count]))\n",
    "logits_3 = tf.matmul(output_2, W_3) + b_3\n",
    "logits = logits_3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss=cross_entropy_loss+tf.add_n(tf.get_collection(\"losses\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainig_step = 50000 #训练50000个step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.train.GradientDescentOptimizer( #定义优化方法为梯度下降\n",
    "    learning_rate=learning_rate).minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 100 training steps, the loss is 83.7485, the validation accuracy is 0.9062\n",
      "after 200 training steps, the loss is 72.3183, the validation accuracy is 0.9158\n",
      "after 300 training steps, the loss is 61.6518, the validation accuracy is 0.9272\n",
      "after 400 training steps, the loss is 52.4096, the validation accuracy is 0.9296\n",
      "after 500 training steps, the loss is 43.9976, the validation accuracy is 0.9164\n",
      "after 600 training steps, the loss is 36.4935, the validation accuracy is 0.9306\n",
      "WARNING:tensorflow:From D:\\Users\\Administrator\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\training\\saver.py:966: remove_checkpoint (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to delete files with this prefix.\n",
      "after 700 training steps, the loss is 30.0015, the validation accuracy is 0.9316\n",
      "after 800 training steps, the loss is 24.4408, the validation accuracy is 0.9316\n",
      "after 900 training steps, the loss is 19.7223, the validation accuracy is 0.9218\n",
      "after 1000 training steps, the loss is 15.6635, the validation accuracy is 0.9108\n",
      "after 1100 training steps, the loss is 12.1011, the validation accuracy is 0.9086\n",
      "after 1200 training steps, the loss is 9.3512, the validation accuracy is 0.9134\n",
      "after 1300 training steps, the loss is 7.11383, the validation accuracy is 0.8936\n",
      "after 1400 training steps, the loss is 5.29468, the validation accuracy is 0.8918\n",
      "after 1500 training steps, the loss is 3.7806, the validation accuracy is 0.9176\n",
      "after 1600 training steps, the loss is 2.84884, the validation accuracy is 0.8962\n",
      "after 1700 training steps, the loss is 2.07233, the validation accuracy is 0.9192\n",
      "after 1800 training steps, the loss is 1.64973, the validation accuracy is 0.8782\n",
      "after 1900 training steps, the loss is 1.19766, the validation accuracy is 0.9194\n",
      "after 2000 training steps, the loss is 0.981929, the validation accuracy is 0.9166\n",
      "after 2100 training steps, the loss is 1.03326, the validation accuracy is 0.9048\n",
      "after 2200 training steps, the loss is 0.840986, the validation accuracy is 0.9158\n",
      "after 2300 training steps, the loss is 0.873604, the validation accuracy is 0.9218\n",
      "after 2400 training steps, the loss is 0.722607, the validation accuracy is 0.9262\n",
      "after 2500 training steps, the loss is 0.739548, the validation accuracy is 0.9252\n",
      "after 2600 training steps, the loss is 0.77387, the validation accuracy is 0.9286\n",
      "after 2700 training steps, the loss is 0.729858, the validation accuracy is 0.932\n",
      "after 2800 training steps, the loss is 0.759361, the validation accuracy is 0.9216\n",
      "after 2900 training steps, the loss is 0.692398, the validation accuracy is 0.9268\n",
      "after 3000 training steps, the loss is 0.737662, the validation accuracy is 0.9298\n",
      "after 3100 training steps, the loss is 0.74349, the validation accuracy is 0.9268\n",
      "after 3200 training steps, the loss is 0.750245, the validation accuracy is 0.9368\n",
      "after 3300 training steps, the loss is 0.662279, the validation accuracy is 0.9282\n",
      "after 3400 training steps, the loss is 0.563781, the validation accuracy is 0.9356\n",
      "after 3500 training steps, the loss is 0.698984, the validation accuracy is 0.9366\n",
      "after 3600 training steps, the loss is 0.740012, the validation accuracy is 0.9144\n",
      "after 3700 training steps, the loss is 0.561162, the validation accuracy is 0.938\n",
      "after 3800 training steps, the loss is 0.696784, the validation accuracy is 0.9388\n",
      "after 3900 training steps, the loss is 0.574304, the validation accuracy is 0.9288\n",
      "after 4000 training steps, the loss is 0.533816, the validation accuracy is 0.9364\n",
      "after 4100 training steps, the loss is 0.654969, the validation accuracy is 0.938\n",
      "after 4200 training steps, the loss is 0.688893, the validation accuracy is 0.9378\n",
      "after 4300 training steps, the loss is 0.548882, the validation accuracy is 0.9354\n",
      "after 4400 training steps, the loss is 0.623263, the validation accuracy is 0.9372\n",
      "after 4500 training steps, the loss is 0.621877, the validation accuracy is 0.9316\n",
      "after 4600 training steps, the loss is 0.534365, the validation accuracy is 0.9414\n",
      "after 4700 training steps, the loss is 0.569552, the validation accuracy is 0.942\n",
      "after 4800 training steps, the loss is 0.54538, the validation accuracy is 0.934\n",
      "after 4900 training steps, the loss is 0.643433, the validation accuracy is 0.9414\n",
      "after 5000 training steps, the loss is 0.536046, the validation accuracy is 0.9424\n",
      "after 5100 training steps, the loss is 0.496057, the validation accuracy is 0.943\n",
      "after 5200 training steps, the loss is 0.599439, the validation accuracy is 0.939\n",
      "after 5300 training steps, the loss is 0.52878, the validation accuracy is 0.9416\n",
      "after 5400 training steps, the loss is 0.589073, the validation accuracy is 0.9436\n",
      "after 5500 training steps, the loss is 0.58818, the validation accuracy is 0.936\n",
      "after 5600 training steps, the loss is 0.540857, the validation accuracy is 0.9404\n",
      "after 5700 training steps, the loss is 0.584298, the validation accuracy is 0.9416\n",
      "after 5800 training steps, the loss is 0.635012, the validation accuracy is 0.9412\n",
      "after 5900 training steps, the loss is 0.595993, the validation accuracy is 0.9392\n",
      "after 6000 training steps, the loss is 0.607764, the validation accuracy is 0.9454\n",
      "after 6100 training steps, the loss is 0.501457, the validation accuracy is 0.9396\n",
      "after 6200 training steps, the loss is 0.564349, the validation accuracy is 0.939\n",
      "after 6300 training steps, the loss is 0.696428, the validation accuracy is 0.9444\n",
      "after 6400 training steps, the loss is 0.586329, the validation accuracy is 0.9344\n",
      "after 6500 training steps, the loss is 0.622511, the validation accuracy is 0.9352\n",
      "after 6600 training steps, the loss is 0.552768, the validation accuracy is 0.941\n",
      "after 6700 training steps, the loss is 0.505501, the validation accuracy is 0.9436\n",
      "after 6800 training steps, the loss is 0.695082, the validation accuracy is 0.9382\n",
      "after 6900 training steps, the loss is 0.550467, the validation accuracy is 0.9366\n",
      "after 7000 training steps, the loss is 0.597448, the validation accuracy is 0.9444\n",
      "after 7100 training steps, the loss is 0.461491, the validation accuracy is 0.9478\n",
      "after 7200 training steps, the loss is 0.509114, the validation accuracy is 0.9432\n",
      "after 7300 training steps, the loss is 0.612922, the validation accuracy is 0.9424\n",
      "after 7400 training steps, the loss is 0.671679, the validation accuracy is 0.935\n",
      "after 7500 training steps, the loss is 0.511261, the validation accuracy is 0.9446\n",
      "after 7600 training steps, the loss is 0.48145, the validation accuracy is 0.9442\n",
      "after 7700 training steps, the loss is 0.463287, the validation accuracy is 0.9434\n",
      "after 7800 training steps, the loss is 0.545758, the validation accuracy is 0.9418\n",
      "after 7900 training steps, the loss is 0.55818, the validation accuracy is 0.9416\n",
      "after 8000 training steps, the loss is 0.576313, the validation accuracy is 0.9456\n",
      "after 8100 training steps, the loss is 0.607373, the validation accuracy is 0.9406\n",
      "after 8200 training steps, the loss is 0.650496, the validation accuracy is 0.9326\n",
      "after 8300 training steps, the loss is 0.526101, the validation accuracy is 0.9434\n",
      "after 8400 training steps, the loss is 0.559263, the validation accuracy is 0.9392\n",
      "after 8500 training steps, the loss is 0.622605, the validation accuracy is 0.9314\n",
      "after 8600 training steps, the loss is 0.667036, the validation accuracy is 0.9368\n",
      "after 8700 training steps, the loss is 0.550429, the validation accuracy is 0.9422\n",
      "after 8800 training steps, the loss is 0.600834, the validation accuracy is 0.9486\n",
      "after 8900 training steps, the loss is 0.432032, the validation accuracy is 0.9462\n",
      "after 9000 training steps, the loss is 0.476559, the validation accuracy is 0.9478\n",
      "after 9100 training steps, the loss is 0.527689, the validation accuracy is 0.9448\n",
      "after 9200 training steps, the loss is 0.50503, the validation accuracy is 0.9454\n",
      "after 9300 training steps, the loss is 0.49017, the validation accuracy is 0.9378\n",
      "after 9400 training steps, the loss is 0.469075, the validation accuracy is 0.944\n",
      "after 9500 training steps, the loss is 0.484235, the validation accuracy is 0.9438\n",
      "after 9600 training steps, the loss is 0.526537, the validation accuracy is 0.9454\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 9700 training steps, the loss is 0.536784, the validation accuracy is 0.9488\n",
      "after 9800 training steps, the loss is 0.567129, the validation accuracy is 0.941\n",
      "after 9900 training steps, the loss is 0.531701, the validation accuracy is 0.9464\n",
      "after 10000 training steps, the loss is 0.535338, the validation accuracy is 0.9472\n",
      "after 10100 training steps, the loss is 0.422309, the validation accuracy is 0.948\n",
      "after 10200 training steps, the loss is 0.502781, the validation accuracy is 0.9446\n",
      "after 10300 training steps, the loss is 0.539444, the validation accuracy is 0.944\n",
      "after 10400 training steps, the loss is 0.515803, the validation accuracy is 0.935\n",
      "after 10500 training steps, the loss is 0.492602, the validation accuracy is 0.9384\n",
      "after 10600 training steps, the loss is 0.48838, the validation accuracy is 0.943\n",
      "after 10700 training steps, the loss is 0.574447, the validation accuracy is 0.9444\n",
      "after 10800 training steps, the loss is 0.497665, the validation accuracy is 0.9412\n",
      "after 10900 training steps, the loss is 0.527421, the validation accuracy is 0.942\n",
      "after 11000 training steps, the loss is 0.523025, the validation accuracy is 0.9472\n",
      "after 11100 training steps, the loss is 0.452845, the validation accuracy is 0.9466\n",
      "after 11200 training steps, the loss is 0.492777, the validation accuracy is 0.9448\n",
      "after 11300 training steps, the loss is 0.523175, the validation accuracy is 0.9376\n",
      "after 11400 training steps, the loss is 0.606542, the validation accuracy is 0.9392\n",
      "after 11500 training steps, the loss is 0.502964, the validation accuracy is 0.9444\n",
      "after 11600 training steps, the loss is 0.586411, the validation accuracy is 0.9342\n",
      "after 11700 training steps, the loss is 0.44831, the validation accuracy is 0.9462\n",
      "after 11800 training steps, the loss is 0.492303, the validation accuracy is 0.9448\n",
      "after 11900 training steps, the loss is 0.458812, the validation accuracy is 0.9476\n",
      "after 12000 training steps, the loss is 0.581001, the validation accuracy is 0.946\n",
      "after 12100 training steps, the loss is 0.523971, the validation accuracy is 0.9464\n",
      "after 12200 training steps, the loss is 0.512653, the validation accuracy is 0.9506\n",
      "after 12300 training steps, the loss is 0.50604, the validation accuracy is 0.948\n",
      "after 12400 training steps, the loss is 0.486242, the validation accuracy is 0.9462\n",
      "after 12500 training steps, the loss is 0.498561, the validation accuracy is 0.9454\n",
      "after 12600 training steps, the loss is 0.614559, the validation accuracy is 0.9382\n",
      "after 12700 training steps, the loss is 0.483713, the validation accuracy is 0.9462\n",
      "after 12800 training steps, the loss is 0.531854, the validation accuracy is 0.946\n",
      "after 12900 training steps, the loss is 0.561987, the validation accuracy is 0.9336\n",
      "after 13000 training steps, the loss is 0.458428, the validation accuracy is 0.9444\n",
      "after 13100 training steps, the loss is 0.6041, the validation accuracy is 0.9318\n",
      "after 13200 training steps, the loss is 0.506554, the validation accuracy is 0.9432\n",
      "after 13300 training steps, the loss is 0.446286, the validation accuracy is 0.944\n",
      "after 13400 training steps, the loss is 0.542272, the validation accuracy is 0.9422\n",
      "after 13500 training steps, the loss is 0.488704, the validation accuracy is 0.9416\n",
      "after 13600 training steps, the loss is 0.642935, the validation accuracy is 0.9414\n",
      "after 13700 training steps, the loss is 0.518871, the validation accuracy is 0.9376\n",
      "after 13800 training steps, the loss is 0.420857, the validation accuracy is 0.9472\n",
      "after 13900 training steps, the loss is 0.478688, the validation accuracy is 0.9436\n",
      "after 14000 training steps, the loss is 0.518525, the validation accuracy is 0.9316\n",
      "after 14100 training steps, the loss is 0.461123, the validation accuracy is 0.947\n",
      "after 14200 training steps, the loss is 0.590098, the validation accuracy is 0.9482\n",
      "after 14300 training steps, the loss is 0.518281, the validation accuracy is 0.9482\n",
      "after 14400 training steps, the loss is 0.475245, the validation accuracy is 0.9278\n",
      "after 14500 training steps, the loss is 0.48702, the validation accuracy is 0.9458\n",
      "after 14600 training steps, the loss is 0.51739, the validation accuracy is 0.9396\n",
      "after 14700 training steps, the loss is 0.519908, the validation accuracy is 0.9384\n",
      "after 14800 training steps, the loss is 0.470012, the validation accuracy is 0.9472\n",
      "after 14900 training steps, the loss is 0.537126, the validation accuracy is 0.9386\n",
      "after 15000 training steps, the loss is 0.569014, the validation accuracy is 0.9456\n",
      "after 15100 training steps, the loss is 0.56764, the validation accuracy is 0.9424\n",
      "after 15200 training steps, the loss is 0.414317, the validation accuracy is 0.9474\n",
      "after 15300 training steps, the loss is 0.52963, the validation accuracy is 0.9406\n",
      "after 15400 training steps, the loss is 0.520201, the validation accuracy is 0.9394\n",
      "after 15500 training steps, the loss is 0.475312, the validation accuracy is 0.9496\n",
      "after 15600 training steps, the loss is 0.52628, the validation accuracy is 0.932\n",
      "after 15700 training steps, the loss is 0.552391, the validation accuracy is 0.9456\n",
      "after 15800 training steps, the loss is 0.526768, the validation accuracy is 0.9208\n",
      "after 15900 training steps, the loss is 0.494169, the validation accuracy is 0.948\n",
      "after 16000 training steps, the loss is 0.556918, the validation accuracy is 0.944\n",
      "after 16100 training steps, the loss is 0.54127, the validation accuracy is 0.9406\n",
      "after 16200 training steps, the loss is 0.572834, the validation accuracy is 0.9452\n",
      "after 16300 training steps, the loss is 0.602859, the validation accuracy is 0.9474\n",
      "after 16400 training steps, the loss is 0.469473, the validation accuracy is 0.9428\n",
      "after 16500 training steps, the loss is 0.523529, the validation accuracy is 0.939\n",
      "after 16600 training steps, the loss is 0.698229, the validation accuracy is 0.8838\n",
      "after 16700 training steps, the loss is 0.530024, the validation accuracy is 0.945\n",
      "after 16800 training steps, the loss is 0.477145, the validation accuracy is 0.944\n",
      "after 16900 training steps, the loss is 0.645252, the validation accuracy is 0.8852\n",
      "after 17000 training steps, the loss is 0.560305, the validation accuracy is 0.9442\n",
      "after 17100 training steps, the loss is 0.457037, the validation accuracy is 0.9464\n",
      "after 17200 training steps, the loss is 0.547119, the validation accuracy is 0.9372\n",
      "after 17300 training steps, the loss is 0.565294, the validation accuracy is 0.9456\n",
      "after 17400 training steps, the loss is 0.537646, the validation accuracy is 0.9384\n",
      "after 17500 training steps, the loss is 0.437402, the validation accuracy is 0.9372\n",
      "after 17600 training steps, the loss is 0.423508, the validation accuracy is 0.9476\n",
      "after 17700 training steps, the loss is 0.498973, the validation accuracy is 0.9444\n",
      "after 17800 training steps, the loss is 0.417968, the validation accuracy is 0.9474\n",
      "after 17900 training steps, the loss is 0.450505, the validation accuracy is 0.9382\n",
      "after 18000 training steps, the loss is 0.507617, the validation accuracy is 0.9474\n",
      "after 18100 training steps, the loss is 0.546597, the validation accuracy is 0.9444\n",
      "after 18200 training steps, the loss is 0.491483, the validation accuracy is 0.9326\n",
      "after 18300 training steps, the loss is 0.461116, the validation accuracy is 0.946\n",
      "after 18400 training steps, the loss is 0.479813, the validation accuracy is 0.948\n",
      "after 18500 training steps, the loss is 0.566547, the validation accuracy is 0.947\n",
      "after 18600 training steps, the loss is 0.593248, the validation accuracy is 0.9418\n",
      "after 18700 training steps, the loss is 0.424912, the validation accuracy is 0.9476\n",
      "after 18800 training steps, the loss is 0.502095, the validation accuracy is 0.9374\n",
      "after 18900 training steps, the loss is 0.440304, the validation accuracy is 0.9444\n",
      "after 19000 training steps, the loss is 0.465153, the validation accuracy is 0.9454\n",
      "after 19100 training steps, the loss is 0.561981, the validation accuracy is 0.9474\n",
      "after 19200 training steps, the loss is 0.479275, the validation accuracy is 0.9436\n",
      "after 19300 training steps, the loss is 0.574387, the validation accuracy is 0.9502\n",
      "after 19400 training steps, the loss is 0.484672, the validation accuracy is 0.9416\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 19500 training steps, the loss is 0.52626, the validation accuracy is 0.9508\n",
      "after 19600 training steps, the loss is 0.47139, the validation accuracy is 0.942\n",
      "after 19700 training steps, the loss is 0.465839, the validation accuracy is 0.9382\n",
      "after 19800 training steps, the loss is 0.470319, the validation accuracy is 0.9458\n",
      "after 19900 training steps, the loss is 0.44641, the validation accuracy is 0.9484\n",
      "after 20000 training steps, the loss is 0.483846, the validation accuracy is 0.9474\n",
      "after 20100 training steps, the loss is 0.520942, the validation accuracy is 0.9454\n",
      "after 20200 training steps, the loss is 0.420256, the validation accuracy is 0.9486\n",
      "after 20300 training steps, the loss is 0.891872, the validation accuracy is 0.9244\n",
      "after 20400 training steps, the loss is 0.471313, the validation accuracy is 0.941\n",
      "after 20500 training steps, the loss is 0.520412, the validation accuracy is 0.9428\n",
      "after 20600 training steps, the loss is 0.476589, the validation accuracy is 0.9464\n",
      "after 20700 training steps, the loss is 0.457568, the validation accuracy is 0.9326\n",
      "after 20800 training steps, the loss is 0.500849, the validation accuracy is 0.9412\n",
      "after 20900 training steps, the loss is 0.42662, the validation accuracy is 0.9458\n",
      "after 21000 training steps, the loss is 0.40809, the validation accuracy is 0.9506\n",
      "after 21100 training steps, the loss is 0.439183, the validation accuracy is 0.945\n",
      "after 21200 training steps, the loss is 0.481818, the validation accuracy is 0.9472\n",
      "after 21300 training steps, the loss is 0.444234, the validation accuracy is 0.9446\n",
      "after 21400 training steps, the loss is 0.482481, the validation accuracy is 0.943\n",
      "after 21500 training steps, the loss is 0.418182, the validation accuracy is 0.9444\n",
      "after 21600 training steps, the loss is 0.654106, the validation accuracy is 0.947\n",
      "after 21700 training steps, the loss is 0.532725, the validation accuracy is 0.937\n",
      "after 21800 training steps, the loss is 0.526244, the validation accuracy is 0.9456\n",
      "after 21900 training steps, the loss is 0.555427, the validation accuracy is 0.9484\n",
      "after 22000 training steps, the loss is 0.435934, the validation accuracy is 0.9444\n",
      "after 22100 training steps, the loss is 0.477253, the validation accuracy is 0.9464\n",
      "after 22200 training steps, the loss is 0.539357, the validation accuracy is 0.9422\n",
      "after 22300 training steps, the loss is 0.473116, the validation accuracy is 0.9458\n",
      "after 22400 training steps, the loss is 0.522903, the validation accuracy is 0.9428\n",
      "after 22500 training steps, the loss is 0.497062, the validation accuracy is 0.9478\n",
      "after 22600 training steps, the loss is 0.472419, the validation accuracy is 0.9472\n",
      "after 22700 training steps, the loss is 0.624756, the validation accuracy is 0.9406\n",
      "after 22800 training steps, the loss is 0.482234, the validation accuracy is 0.946\n",
      "after 22900 training steps, the loss is 0.5071, the validation accuracy is 0.9372\n",
      "after 23000 training steps, the loss is 0.666085, the validation accuracy is 0.94\n",
      "after 23100 training steps, the loss is 0.586941, the validation accuracy is 0.934\n",
      "after 23200 training steps, the loss is 0.805892, the validation accuracy is 0.9334\n",
      "after 23300 training steps, the loss is 0.458463, the validation accuracy is 0.9386\n",
      "after 23400 training steps, the loss is 0.594445, the validation accuracy is 0.9472\n",
      "after 23500 training steps, the loss is 0.547209, the validation accuracy is 0.944\n",
      "after 23600 training steps, the loss is 0.471872, the validation accuracy is 0.946\n",
      "after 23700 training steps, the loss is 0.529583, the validation accuracy is 0.9498\n",
      "after 23800 training steps, the loss is 0.466943, the validation accuracy is 0.9456\n",
      "after 23900 training steps, the loss is 0.60159, the validation accuracy is 0.93\n",
      "after 24000 training steps, the loss is 0.499654, the validation accuracy is 0.9334\n",
      "after 24100 training steps, the loss is 0.45241, the validation accuracy is 0.948\n",
      "after 24200 training steps, the loss is 0.571927, the validation accuracy is 0.9428\n",
      "after 24300 training steps, the loss is 0.430112, the validation accuracy is 0.9492\n",
      "after 24400 training steps, the loss is 0.433905, the validation accuracy is 0.9456\n",
      "after 24500 training steps, the loss is 0.518151, the validation accuracy is 0.9372\n",
      "after 24600 training steps, the loss is 0.603081, the validation accuracy is 0.9248\n",
      "after 24700 training steps, the loss is 0.442692, the validation accuracy is 0.9466\n",
      "after 24800 training steps, the loss is 0.490028, the validation accuracy is 0.9388\n",
      "after 24900 training steps, the loss is 0.608507, the validation accuracy is 0.9426\n",
      "after 25000 training steps, the loss is 0.559004, the validation accuracy is 0.9448\n",
      "after 25100 training steps, the loss is 0.544244, the validation accuracy is 0.9432\n",
      "after 25200 training steps, the loss is 0.489899, the validation accuracy is 0.9382\n",
      "after 25300 training steps, the loss is 0.421015, the validation accuracy is 0.9388\n",
      "after 25400 training steps, the loss is 0.708511, the validation accuracy is 0.9444\n",
      "after 25500 training steps, the loss is 0.622622, the validation accuracy is 0.9196\n",
      "after 25600 training steps, the loss is 0.462208, the validation accuracy is 0.9474\n",
      "after 25700 training steps, the loss is 0.580828, the validation accuracy is 0.9386\n",
      "after 25800 training steps, the loss is 0.46121, the validation accuracy is 0.9474\n",
      "after 25900 training steps, the loss is 0.464861, the validation accuracy is 0.9456\n",
      "after 26000 training steps, the loss is 0.490598, the validation accuracy is 0.9432\n",
      "after 26100 training steps, the loss is 0.520746, the validation accuracy is 0.9432\n",
      "after 26200 training steps, the loss is 0.644742, the validation accuracy is 0.9298\n",
      "after 26300 training steps, the loss is 0.581652, the validation accuracy is 0.9482\n",
      "after 26400 training steps, the loss is 0.49221, the validation accuracy is 0.939\n",
      "after 26500 training steps, the loss is 0.446825, the validation accuracy is 0.9464\n",
      "after 26600 training steps, the loss is 0.584186, the validation accuracy is 0.9386\n",
      "after 26700 training steps, the loss is 0.411028, the validation accuracy is 0.9502\n",
      "after 26800 training steps, the loss is 0.407765, the validation accuracy is 0.948\n",
      "after 26900 training steps, the loss is 0.434921, the validation accuracy is 0.9492\n",
      "after 27000 training steps, the loss is 0.609822, the validation accuracy is 0.9186\n",
      "after 27100 training steps, the loss is 0.413976, the validation accuracy is 0.9474\n",
      "after 27200 training steps, the loss is 0.530642, the validation accuracy is 0.941\n",
      "after 27300 training steps, the loss is 0.423133, the validation accuracy is 0.9444\n",
      "after 27400 training steps, the loss is 0.650078, the validation accuracy is 0.9288\n",
      "after 27500 training steps, the loss is 0.665858, the validation accuracy is 0.9306\n",
      "after 27600 training steps, the loss is 0.560587, the validation accuracy is 0.936\n",
      "after 27700 training steps, the loss is 0.595306, the validation accuracy is 0.9468\n",
      "after 27800 training steps, the loss is 0.504892, the validation accuracy is 0.9462\n",
      "after 27900 training steps, the loss is 0.453584, the validation accuracy is 0.944\n",
      "after 28000 training steps, the loss is 0.533705, the validation accuracy is 0.949\n",
      "after 28100 training steps, the loss is 0.497242, the validation accuracy is 0.9436\n",
      "after 28200 training steps, the loss is 0.659208, the validation accuracy is 0.9436\n",
      "after 28300 training steps, the loss is 0.404857, the validation accuracy is 0.9448\n",
      "after 28400 training steps, the loss is 0.441333, the validation accuracy is 0.9486\n",
      "after 28500 training steps, the loss is 0.459294, the validation accuracy is 0.9436\n",
      "after 28600 training steps, the loss is 0.45737, the validation accuracy is 0.9408\n",
      "after 28700 training steps, the loss is 0.493772, the validation accuracy is 0.9462\n",
      "after 28800 training steps, the loss is 0.470854, the validation accuracy is 0.9472\n",
      "after 28900 training steps, the loss is 0.581206, the validation accuracy is 0.9436\n",
      "after 29000 training steps, the loss is 0.578996, the validation accuracy is 0.941\n",
      "after 29100 training steps, the loss is 0.513861, the validation accuracy is 0.9428\n",
      "after 29200 training steps, the loss is 0.433021, the validation accuracy is 0.9412\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 29300 training steps, the loss is 0.4469, the validation accuracy is 0.945\n",
      "after 29400 training steps, the loss is 0.543341, the validation accuracy is 0.9492\n",
      "after 29500 training steps, the loss is 0.584729, the validation accuracy is 0.947\n",
      "after 29600 training steps, the loss is 0.44737, the validation accuracy is 0.9454\n",
      "after 29700 training steps, the loss is 0.429583, the validation accuracy is 0.9478\n",
      "after 29800 training steps, the loss is 0.484843, the validation accuracy is 0.9486\n",
      "after 29900 training steps, the loss is 0.497822, the validation accuracy is 0.9432\n",
      "after 30000 training steps, the loss is 0.518776, the validation accuracy is 0.9402\n",
      "after 30100 training steps, the loss is 0.517372, the validation accuracy is 0.939\n",
      "after 30200 training steps, the loss is 0.515369, the validation accuracy is 0.9446\n",
      "after 30300 training steps, the loss is 0.411426, the validation accuracy is 0.9384\n",
      "after 30400 training steps, the loss is 0.533024, the validation accuracy is 0.9364\n",
      "after 30500 training steps, the loss is 0.506569, the validation accuracy is 0.9462\n",
      "after 30600 training steps, the loss is 0.472564, the validation accuracy is 0.949\n",
      "after 30700 training steps, the loss is 0.547217, the validation accuracy is 0.9456\n",
      "after 30800 training steps, the loss is 0.525529, the validation accuracy is 0.9486\n",
      "after 30900 training steps, the loss is 0.497301, the validation accuracy is 0.9364\n",
      "after 31000 training steps, the loss is 0.425109, the validation accuracy is 0.95\n",
      "after 31100 training steps, the loss is 0.50542, the validation accuracy is 0.9402\n",
      "after 31200 training steps, the loss is 0.464132, the validation accuracy is 0.9384\n",
      "after 31300 training steps, the loss is 0.541939, the validation accuracy is 0.9178\n",
      "after 31400 training steps, the loss is 0.558923, the validation accuracy is 0.944\n",
      "after 31500 training steps, the loss is 0.54881, the validation accuracy is 0.9308\n",
      "after 31600 training steps, the loss is 0.488759, the validation accuracy is 0.9472\n",
      "after 31700 training steps, the loss is 0.544562, the validation accuracy is 0.9472\n",
      "after 31800 training steps, the loss is 0.509648, the validation accuracy is 0.9502\n",
      "after 31900 training steps, the loss is 0.456026, the validation accuracy is 0.9512\n",
      "after 32000 training steps, the loss is 0.608135, the validation accuracy is 0.9444\n",
      "after 32100 training steps, the loss is 0.619665, the validation accuracy is 0.9478\n",
      "after 32200 training steps, the loss is 0.512223, the validation accuracy is 0.9476\n",
      "after 32300 training steps, the loss is 0.547756, the validation accuracy is 0.9396\n",
      "after 32400 training steps, the loss is 0.618118, the validation accuracy is 0.9388\n",
      "after 32500 training steps, the loss is 0.564823, the validation accuracy is 0.9388\n",
      "after 32600 training steps, the loss is 0.440314, the validation accuracy is 0.947\n",
      "after 32700 training steps, the loss is 0.61595, the validation accuracy is 0.9436\n",
      "after 32800 training steps, the loss is 0.38812, the validation accuracy is 0.9506\n",
      "after 32900 training steps, the loss is 0.370818, the validation accuracy is 0.9494\n",
      "after 33000 training steps, the loss is 0.388251, the validation accuracy is 0.9492\n",
      "after 33100 training steps, the loss is 0.416946, the validation accuracy is 0.9458\n",
      "after 33200 training steps, the loss is 0.429445, the validation accuracy is 0.9478\n",
      "after 33300 training steps, the loss is 0.54161, the validation accuracy is 0.9454\n",
      "after 33400 training steps, the loss is 0.501869, the validation accuracy is 0.9444\n",
      "after 33500 training steps, the loss is 0.543359, the validation accuracy is 0.9488\n",
      "after 33600 training steps, the loss is 0.498701, the validation accuracy is 0.9366\n",
      "after 33700 training steps, the loss is 0.399206, the validation accuracy is 0.9456\n",
      "after 33800 training steps, the loss is 0.514371, the validation accuracy is 0.9444\n",
      "after 33900 training steps, the loss is 0.561687, the validation accuracy is 0.9164\n",
      "after 34000 training steps, the loss is 0.483621, the validation accuracy is 0.947\n",
      "after 34100 training steps, the loss is 0.575552, the validation accuracy is 0.9442\n",
      "after 34200 training steps, the loss is 0.471652, the validation accuracy is 0.9444\n",
      "after 34300 training steps, the loss is 0.431264, the validation accuracy is 0.9386\n",
      "after 34400 training steps, the loss is 0.477575, the validation accuracy is 0.9434\n",
      "after 34500 training steps, the loss is 0.413469, the validation accuracy is 0.9478\n",
      "after 34600 training steps, the loss is 0.556202, the validation accuracy is 0.942\n",
      "after 34700 training steps, the loss is 0.47594, the validation accuracy is 0.939\n",
      "after 34800 training steps, the loss is 0.536432, the validation accuracy is 0.935\n",
      "after 34900 training steps, the loss is 0.377063, the validation accuracy is 0.9494\n",
      "after 35000 training steps, the loss is 0.449891, the validation accuracy is 0.9456\n",
      "after 35100 training steps, the loss is 0.393329, the validation accuracy is 0.9468\n",
      "after 35200 training steps, the loss is 0.44106, the validation accuracy is 0.9536\n",
      "after 35300 training steps, the loss is 0.457403, the validation accuracy is 0.951\n",
      "after 35400 training steps, the loss is 0.486199, the validation accuracy is 0.9324\n",
      "after 35500 training steps, the loss is 0.607076, the validation accuracy is 0.925\n",
      "after 35600 training steps, the loss is 0.427987, the validation accuracy is 0.95\n",
      "after 35700 training steps, the loss is 0.476138, the validation accuracy is 0.9504\n",
      "after 35800 training steps, the loss is 0.432357, the validation accuracy is 0.9458\n",
      "after 35900 training steps, the loss is 0.505847, the validation accuracy is 0.9344\n",
      "after 36000 training steps, the loss is 0.50423, the validation accuracy is 0.9474\n",
      "after 36100 training steps, the loss is 0.391384, the validation accuracy is 0.9444\n",
      "after 36200 training steps, the loss is 0.432115, the validation accuracy is 0.9434\n",
      "after 36300 training steps, the loss is 0.487008, the validation accuracy is 0.9474\n",
      "after 36400 training steps, the loss is 0.483991, the validation accuracy is 0.9482\n",
      "after 36500 training steps, the loss is 0.660873, the validation accuracy is 0.9376\n",
      "after 36600 training steps, the loss is 0.50907, the validation accuracy is 0.951\n",
      "after 36700 training steps, the loss is 0.449239, the validation accuracy is 0.9492\n",
      "after 36800 training steps, the loss is 0.411226, the validation accuracy is 0.9438\n",
      "after 36900 training steps, the loss is 0.515185, the validation accuracy is 0.9406\n",
      "after 37000 training steps, the loss is 0.446758, the validation accuracy is 0.9494\n",
      "after 37100 training steps, the loss is 0.514847, the validation accuracy is 0.9436\n",
      "after 37200 training steps, the loss is 0.524403, the validation accuracy is 0.92\n",
      "after 37300 training steps, the loss is 0.486476, the validation accuracy is 0.9488\n",
      "after 37400 training steps, the loss is 0.474713, the validation accuracy is 0.9436\n",
      "after 37500 training steps, the loss is 0.539917, the validation accuracy is 0.9512\n",
      "after 37600 training steps, the loss is 0.472484, the validation accuracy is 0.9472\n",
      "after 37700 training steps, the loss is 0.365724, the validation accuracy is 0.9508\n",
      "after 37800 training steps, the loss is 0.38483, the validation accuracy is 0.9468\n",
      "after 37900 training steps, the loss is 0.518788, the validation accuracy is 0.9396\n",
      "after 38000 training steps, the loss is 0.387384, the validation accuracy is 0.9456\n",
      "after 38100 training steps, the loss is 0.43563, the validation accuracy is 0.9446\n",
      "after 38200 training steps, the loss is 0.506158, the validation accuracy is 0.9492\n",
      "after 38300 training steps, the loss is 0.495865, the validation accuracy is 0.954\n",
      "after 38400 training steps, the loss is 0.423034, the validation accuracy is 0.948\n",
      "after 38500 training steps, the loss is 0.427752, the validation accuracy is 0.9472\n",
      "after 38600 training steps, the loss is 0.422432, the validation accuracy is 0.9452\n",
      "after 38700 training steps, the loss is 0.421133, the validation accuracy is 0.9426\n",
      "after 38800 training steps, the loss is 0.558346, the validation accuracy is 0.9482\n",
      "after 38900 training steps, the loss is 0.473175, the validation accuracy is 0.946\n",
      "after 39000 training steps, the loss is 0.62044, the validation accuracy is 0.9322\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 39100 training steps, the loss is 0.456851, the validation accuracy is 0.949\n",
      "after 39200 training steps, the loss is 0.471963, the validation accuracy is 0.944\n",
      "after 39300 training steps, the loss is 0.445885, the validation accuracy is 0.9536\n",
      "after 39400 training steps, the loss is 0.455421, the validation accuracy is 0.948\n",
      "after 39500 training steps, the loss is 0.540202, the validation accuracy is 0.9452\n",
      "after 39600 training steps, the loss is 0.460466, the validation accuracy is 0.9474\n",
      "after 39700 training steps, the loss is 0.366238, the validation accuracy is 0.946\n",
      "after 39800 training steps, the loss is 0.490626, the validation accuracy is 0.937\n",
      "after 39900 training steps, the loss is 0.527793, the validation accuracy is 0.95\n",
      "after 40000 training steps, the loss is 0.475629, the validation accuracy is 0.9466\n",
      "after 40100 training steps, the loss is 0.610187, the validation accuracy is 0.9488\n",
      "after 40200 training steps, the loss is 0.454232, the validation accuracy is 0.9468\n",
      "after 40300 training steps, the loss is 0.463899, the validation accuracy is 0.9506\n",
      "after 40400 training steps, the loss is 0.461641, the validation accuracy is 0.9476\n",
      "after 40500 training steps, the loss is 0.428157, the validation accuracy is 0.9492\n",
      "after 40600 training steps, the loss is 0.605648, the validation accuracy is 0.9358\n",
      "after 40700 training steps, the loss is 0.445471, the validation accuracy is 0.9438\n",
      "after 40800 training steps, the loss is 0.442369, the validation accuracy is 0.947\n",
      "after 40900 training steps, the loss is 0.449364, the validation accuracy is 0.941\n",
      "after 41000 training steps, the loss is 0.372894, the validation accuracy is 0.9512\n",
      "after 41100 training steps, the loss is 0.470943, the validation accuracy is 0.9428\n",
      "after 41200 training steps, the loss is 0.558409, the validation accuracy is 0.938\n",
      "after 41300 training steps, the loss is 0.465077, the validation accuracy is 0.9502\n",
      "after 41400 training steps, the loss is 0.386466, the validation accuracy is 0.9498\n",
      "after 41500 training steps, the loss is 0.544914, the validation accuracy is 0.9428\n",
      "after 41600 training steps, the loss is 0.496139, the validation accuracy is 0.9446\n",
      "after 41700 training steps, the loss is 0.42323, the validation accuracy is 0.949\n",
      "after 41800 training steps, the loss is 0.505462, the validation accuracy is 0.931\n",
      "after 41900 training steps, the loss is 0.447982, the validation accuracy is 0.9492\n",
      "after 42000 training steps, the loss is 0.456846, the validation accuracy is 0.9348\n",
      "after 42100 training steps, the loss is 0.498799, the validation accuracy is 0.9446\n",
      "after 42200 training steps, the loss is 0.619888, the validation accuracy is 0.9206\n",
      "after 42300 training steps, the loss is 0.579416, the validation accuracy is 0.9414\n",
      "after 42400 training steps, the loss is 0.415515, the validation accuracy is 0.94\n",
      "after 42500 training steps, the loss is 0.410478, the validation accuracy is 0.941\n",
      "after 42600 training steps, the loss is 0.500947, the validation accuracy is 0.9382\n",
      "after 42700 training steps, the loss is 0.429692, the validation accuracy is 0.9446\n",
      "after 42800 training steps, the loss is 0.43956, the validation accuracy is 0.9482\n",
      "after 42900 training steps, the loss is 0.503617, the validation accuracy is 0.9428\n",
      "after 43000 training steps, the loss is 0.528611, the validation accuracy is 0.9394\n",
      "after 43100 training steps, the loss is 0.568823, the validation accuracy is 0.9466\n",
      "after 43200 training steps, the loss is 0.513162, the validation accuracy is 0.9504\n",
      "after 43300 training steps, the loss is 0.488203, the validation accuracy is 0.949\n",
      "after 43400 training steps, the loss is 0.419622, the validation accuracy is 0.947\n",
      "after 43500 training steps, the loss is 0.495969, the validation accuracy is 0.9432\n",
      "after 43600 training steps, the loss is 0.569077, the validation accuracy is 0.94\n",
      "after 43700 training steps, the loss is 0.439733, the validation accuracy is 0.9398\n",
      "after 43800 training steps, the loss is 0.471085, the validation accuracy is 0.9402\n",
      "after 43900 training steps, the loss is 0.451284, the validation accuracy is 0.9472\n",
      "after 44000 training steps, the loss is 0.430452, the validation accuracy is 0.9492\n",
      "after 44100 training steps, the loss is 0.44455, the validation accuracy is 0.9424\n",
      "after 44200 training steps, the loss is 0.38027, the validation accuracy is 0.9476\n",
      "after 44300 training steps, the loss is 0.634853, the validation accuracy is 0.9522\n",
      "after 44400 training steps, the loss is 0.377721, the validation accuracy is 0.9502\n",
      "after 44500 training steps, the loss is 0.451828, the validation accuracy is 0.9518\n",
      "after 44600 training steps, the loss is 0.52298, the validation accuracy is 0.9384\n",
      "after 44700 training steps, the loss is 0.376654, the validation accuracy is 0.9474\n",
      "after 44800 training steps, the loss is 0.589107, the validation accuracy is 0.9432\n",
      "after 44900 training steps, the loss is 0.456948, the validation accuracy is 0.9428\n",
      "after 45000 training steps, the loss is 0.49432, the validation accuracy is 0.9338\n",
      "after 45100 training steps, the loss is 0.371406, the validation accuracy is 0.9506\n",
      "after 45200 training steps, the loss is 0.533981, the validation accuracy is 0.9374\n",
      "after 45300 training steps, the loss is 0.408956, the validation accuracy is 0.9482\n",
      "after 45400 training steps, the loss is 0.436696, the validation accuracy is 0.9468\n",
      "after 45500 training steps, the loss is 0.525705, the validation accuracy is 0.9502\n",
      "after 45600 training steps, the loss is 0.579644, the validation accuracy is 0.9442\n",
      "after 45700 training steps, the loss is 0.455044, the validation accuracy is 0.9458\n",
      "after 45800 training steps, the loss is 0.479392, the validation accuracy is 0.9476\n",
      "after 45900 training steps, the loss is 0.431609, the validation accuracy is 0.943\n",
      "after 46000 training steps, the loss is 0.638683, the validation accuracy is 0.9396\n",
      "after 46100 training steps, the loss is 0.527698, the validation accuracy is 0.9466\n",
      "after 46200 training steps, the loss is 0.425455, the validation accuracy is 0.9516\n",
      "after 46300 training steps, the loss is 0.549871, the validation accuracy is 0.9408\n",
      "after 46400 training steps, the loss is 0.461035, the validation accuracy is 0.951\n",
      "after 46500 training steps, the loss is 0.491873, the validation accuracy is 0.9288\n",
      "after 46600 training steps, the loss is 0.4545, the validation accuracy is 0.9406\n",
      "after 46700 training steps, the loss is 0.514137, the validation accuracy is 0.9484\n",
      "after 46800 training steps, the loss is 0.421409, the validation accuracy is 0.9452\n",
      "after 46900 training steps, the loss is 0.485921, the validation accuracy is 0.9328\n",
      "after 47000 training steps, the loss is 0.494844, the validation accuracy is 0.9472\n",
      "after 47100 training steps, the loss is 0.439252, the validation accuracy is 0.942\n",
      "after 47200 training steps, the loss is 0.460404, the validation accuracy is 0.9492\n",
      "after 47300 training steps, the loss is 0.427135, the validation accuracy is 0.951\n",
      "after 47400 training steps, the loss is 0.524366, the validation accuracy is 0.945\n",
      "after 47500 training steps, the loss is 0.44847, the validation accuracy is 0.9446\n",
      "after 47600 training steps, the loss is 0.350202, the validation accuracy is 0.9516\n",
      "after 47700 training steps, the loss is 0.440559, the validation accuracy is 0.948\n",
      "after 47800 training steps, the loss is 0.507501, the validation accuracy is 0.9426\n",
      "after 47900 training steps, the loss is 0.414107, the validation accuracy is 0.9492\n",
      "after 48000 training steps, the loss is 0.54462, the validation accuracy is 0.9462\n",
      "after 48100 training steps, the loss is 0.546673, the validation accuracy is 0.938\n",
      "after 48200 training steps, the loss is 0.437114, the validation accuracy is 0.9496\n",
      "after 48300 training steps, the loss is 0.443931, the validation accuracy is 0.9472\n",
      "after 48400 training steps, the loss is 0.431541, the validation accuracy is 0.9488\n",
      "after 48500 training steps, the loss is 0.532155, the validation accuracy is 0.948\n",
      "after 48600 training steps, the loss is 0.514662, the validation accuracy is 0.9386\n",
      "after 48700 training steps, the loss is 0.54637, the validation accuracy is 0.9516\n",
      "after 48800 training steps, the loss is 0.44254, the validation accuracy is 0.9488\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 48900 training steps, the loss is 0.444387, the validation accuracy is 0.9516\n",
      "after 49000 training steps, the loss is 0.50373, the validation accuracy is 0.9496\n",
      "after 49100 training steps, the loss is 0.480311, the validation accuracy is 0.9396\n",
      "after 49200 training steps, the loss is 0.561284, the validation accuracy is 0.928\n",
      "after 49300 training steps, the loss is 0.479973, the validation accuracy is 0.9482\n",
      "after 49400 training steps, the loss is 0.633399, the validation accuracy is 0.9492\n",
      "after 49500 training steps, the loss is 0.650074, the validation accuracy is 0.928\n",
      "after 49600 training steps, the loss is 0.371699, the validation accuracy is 0.945\n",
      "after 49700 training steps, the loss is 0.578156, the validation accuracy is 0.9504\n",
      "after 49800 training steps, the loss is 0.517, the validation accuracy is 0.9474\n",
      "after 49900 training steps, the loss is 0.464287, the validation accuracy is 0.9448\n",
      "the training is finish!\n",
      "the test accuarcy is: 0.9476\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "        #定义验证集与测试集\n",
    "    validate_data = {\n",
    "        x: mnist.validation.images,\n",
    "        y: mnist.validation.labels,\n",
    "    }\n",
    "    test_data = {x: mnist.test.images, y: mnist.test.labels}\n",
    "\n",
    "    for i in range(trainig_step):\n",
    "        xs, ys = mnist.train.next_batch(batch_size)\n",
    "        _, loss_value = sess.run(\n",
    "            [optimizer, loss],\n",
    "            feed_dict={\n",
    "                x: xs,\n",
    "                y: ys,\n",
    "                learning_rate: 0.1\n",
    "            })\n",
    "\n",
    "        #每100次训练打印一次损失值与验证准确率\n",
    "        if i > 0 and i % 100 == 0:\n",
    "            validate_accuracy = sess.run(accuracy, feed_dict=validate_data)\n",
    "            print(\n",
    "                \"after %d training steps, the loss is %g, the validation accuracy is %g\"\n",
    "                % (i, loss_value, validate_accuracy))\n",
    "            saver.save(sess, './model.ckpt', global_step=i)\n",
    "\n",
    "    print(\"the training is finish!\")\n",
    "    #最终的测试准确率\n",
    "    acc = sess.run(accuracy, feed_dict=test_data)\n",
    "    print(\"the test accuarcy is:\", acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "加入了l1正则以后，收敛的速度明显变慢了，但是效果变得稳定了。中间出现了无论怎样增加训练轮数都无法提升效果的状况，推测可能是学习率设的太大，发生了震荡，也有可能是隐层只有一个，复杂度不够.下面试一试l2正则"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_weight(shape, lambda1):\n",
    "    var = tf.Variable(tf.truncated_normal(shape,stddev=0.1), dtype=tf.float32) # 生成一个变量\n",
    "    tf.add_to_collection('losses', tf.contrib.layers.l2_regularizer(lambda1)(var)) # add_to_collection()函数将新生成变量的L1正则化损失加入集合losses\n",
    "    return var"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "L1_units_count = 500 #第一层神经元的个数\n",
    "W_1 = get_weight([784, L1_units_count],0.001) #variable主要用于数据存储,在计算图的运算过程中，其值会一直保存到程序运行结束\n",
    "b_1 = tf.Variable(initialize([L1_units_count]))#比如神经网络中的权重和bias等，在训练过后，总是希望这些参数能够保存下来，而不是直接就消失了，所以这个时候要用到Variable\n",
    "logits_1 = tf.matmul(x, W_1) + b_1 #logits函数，即y=wx+b在tensorflow中计算图的定义方法\n",
    "output_1 = tf.nn.relu(logits_1) #激活函数为relu\n",
    "L2_units_count =300 #增加隐层\n",
    "W_2 = get_weight([L1_units_count, L2_units_count],0.001)\n",
    "b_2 = tf.Variable(initialize([L2_units_count]))\n",
    "logits_2 = tf.matmul(output_1, W_2) + b_2  \n",
    "output_2 = tf.nn.relu(logits_2) #激活函数为relu\n",
    "L3_units_count = 10 #输出层\n",
    "W_3 = get_weight([L2_units_count, L3_units_count],0.001)\n",
    "b_3 = tf.Variable(initialize([L3_units_count]))\n",
    "logits_3 = tf.matmul(output_2, W_3) + b_3\n",
    "logits = logits_3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "cross_entropy_loss = tf.reduce_mean(\n",
    "    tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=y))#损失函数是交叉熵损失\n",
    "\n",
    "\n",
    "batch_size = 100 #一轮喂入的数据为100个\n",
    "trainig_step = 20000 #训练10000个step\n",
    "\n",
    "saver = tf.train.Saver()#模型保存\n",
    "pred = tf.nn.softmax(logits)# 原始logit用softmax转换一下\n",
    "correct_pred = tf.equal(tf.argmax(pred, 1), y)# 预测的准确率\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))#取平均值得到最终的准确率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss=cross_entropy_loss+tf.add_n(tf.get_collection(\"losses\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.train.GradientDescentOptimizer( #定义优化方法为梯度下降\n",
    "    learning_rate=learning_rate).minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 100 training steps, the loss is 270.293, the validation accuracy is 0.8938\n",
      "after 200 training steps, the loss is 264.835, the validation accuracy is 0.9276\n",
      "after 300 training steps, the loss is 259.473, the validation accuracy is 0.9386\n",
      "after 400 training steps, the loss is 254.348, the validation accuracy is 0.9438\n",
      "after 500 training steps, the loss is 249.357, the validation accuracy is 0.951\n",
      "after 600 training steps, the loss is 244.421, the validation accuracy is 0.9508\n",
      "WARNING:tensorflow:From D:\\Users\\Administrator\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\training\\saver.py:966: remove_checkpoint (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to delete files with this prefix.\n",
      "after 700 training steps, the loss is 239.63, the validation accuracy is 0.9518\n",
      "after 800 training steps, the loss is 234.864, the validation accuracy is 0.9572\n",
      "after 900 training steps, the loss is 230.127, the validation accuracy is 0.9608\n",
      "after 1000 training steps, the loss is 225.574, the validation accuracy is 0.9612\n",
      "after 1100 training steps, the loss is 221.058, the validation accuracy is 0.963\n",
      "after 1200 training steps, the loss is 216.667, the validation accuracy is 0.9654\n",
      "after 1300 training steps, the loss is 212.413, the validation accuracy is 0.967\n",
      "after 1400 training steps, the loss is 208.221, the validation accuracy is 0.9646\n",
      "after 1500 training steps, the loss is 204.194, the validation accuracy is 0.9682\n",
      "after 1600 training steps, the loss is 200.064, the validation accuracy is 0.959\n",
      "after 1700 training steps, the loss is 196.12, the validation accuracy is 0.9682\n",
      "after 1800 training steps, the loss is 192.244, the validation accuracy is 0.9704\n",
      "after 1900 training steps, the loss is 188.391, the validation accuracy is 0.969\n",
      "after 2000 training steps, the loss is 184.671, the validation accuracy is 0.969\n",
      "after 2100 training steps, the loss is 181.037, the validation accuracy is 0.9714\n",
      "after 2200 training steps, the loss is 177.408, the validation accuracy is 0.972\n",
      "after 2300 training steps, the loss is 173.913, the validation accuracy is 0.9714\n",
      "after 2400 training steps, the loss is 170.437, the validation accuracy is 0.972\n",
      "after 2500 training steps, the loss is 167.073, the validation accuracy is 0.9724\n",
      "after 2600 training steps, the loss is 163.765, the validation accuracy is 0.975\n",
      "after 2700 training steps, the loss is 160.552, the validation accuracy is 0.9736\n",
      "after 2800 training steps, the loss is 157.437, the validation accuracy is 0.9736\n",
      "after 2900 training steps, the loss is 154.299, the validation accuracy is 0.9748\n",
      "after 3000 training steps, the loss is 151.192, the validation accuracy is 0.9762\n",
      "after 3100 training steps, the loss is 148.18, the validation accuracy is 0.9742\n",
      "after 3200 training steps, the loss is 145.321, the validation accuracy is 0.9738\n",
      "after 3300 training steps, the loss is 142.39, the validation accuracy is 0.9748\n",
      "after 3400 training steps, the loss is 139.577, the validation accuracy is 0.975\n",
      "after 3500 training steps, the loss is 136.873, the validation accuracy is 0.9738\n",
      "after 3600 training steps, the loss is 134.099, the validation accuracy is 0.9758\n",
      "after 3700 training steps, the loss is 131.469, the validation accuracy is 0.9756\n",
      "after 3800 training steps, the loss is 128.844, the validation accuracy is 0.975\n",
      "after 3900 training steps, the loss is 126.336, the validation accuracy is 0.9764\n",
      "after 4000 training steps, the loss is 123.815, the validation accuracy is 0.9756\n",
      "after 4100 training steps, the loss is 121.442, the validation accuracy is 0.977\n",
      "after 4200 training steps, the loss is 118.977, the validation accuracy is 0.9746\n",
      "after 4300 training steps, the loss is 116.611, the validation accuracy is 0.9786\n",
      "after 4400 training steps, the loss is 114.292, the validation accuracy is 0.977\n",
      "after 4500 training steps, the loss is 112.064, the validation accuracy is 0.9778\n",
      "after 4600 training steps, the loss is 109.841, the validation accuracy is 0.9752\n",
      "after 4700 training steps, the loss is 107.622, the validation accuracy is 0.9772\n",
      "after 4800 training steps, the loss is 105.51, the validation accuracy is 0.9772\n",
      "after 4900 training steps, the loss is 103.449, the validation accuracy is 0.9766\n",
      "after 5000 training steps, the loss is 101.429, the validation accuracy is 0.9778\n",
      "after 5100 training steps, the loss is 99.3721, the validation accuracy is 0.9772\n",
      "after 5200 training steps, the loss is 97.4127, the validation accuracy is 0.9784\n",
      "after 5300 training steps, the loss is 95.5303, the validation accuracy is 0.9768\n",
      "after 5400 training steps, the loss is 93.6032, the validation accuracy is 0.978\n",
      "after 5500 training steps, the loss is 91.743, the validation accuracy is 0.9792\n",
      "after 5600 training steps, the loss is 89.9243, the validation accuracy is 0.9766\n",
      "after 5700 training steps, the loss is 88.1725, the validation accuracy is 0.979\n",
      "after 5800 training steps, the loss is 86.4114, the validation accuracy is 0.9796\n",
      "after 5900 training steps, the loss is 84.7099, the validation accuracy is 0.9762\n",
      "after 6000 training steps, the loss is 83.062, the validation accuracy is 0.9782\n",
      "after 6100 training steps, the loss is 81.3755, the validation accuracy is 0.979\n",
      "after 6200 training steps, the loss is 79.7626, the validation accuracy is 0.9786\n",
      "after 6300 training steps, the loss is 78.1824, the validation accuracy is 0.9786\n",
      "after 6400 training steps, the loss is 76.8309, the validation accuracy is 0.9774\n",
      "after 6500 training steps, the loss is 75.2063, the validation accuracy is 0.9772\n",
      "after 6600 training steps, the loss is 73.6373, the validation accuracy is 0.98\n",
      "after 6700 training steps, the loss is 72.1988, the validation accuracy is 0.9796\n",
      "after 6800 training steps, the loss is 70.7755, the validation accuracy is 0.9796\n",
      "after 6900 training steps, the loss is 69.4, the validation accuracy is 0.9802\n",
      "after 7000 training steps, the loss is 68.0039, the validation accuracy is 0.9778\n",
      "after 7100 training steps, the loss is 66.6907, the validation accuracy is 0.977\n",
      "after 7200 training steps, the loss is 65.3387, the validation accuracy is 0.9802\n",
      "after 7300 training steps, the loss is 64.0708, the validation accuracy is 0.9792\n",
      "after 7400 training steps, the loss is 62.7598, the validation accuracy is 0.9792\n",
      "after 7500 training steps, the loss is 61.5888, the validation accuracy is 0.979\n",
      "after 7600 training steps, the loss is 60.3199, the validation accuracy is 0.9804\n",
      "after 7700 training steps, the loss is 59.134, the validation accuracy is 0.9768\n",
      "after 7800 training steps, the loss is 57.9768, the validation accuracy is 0.9788\n",
      "after 7900 training steps, the loss is 56.8253, the validation accuracy is 0.9802\n",
      "after 8000 training steps, the loss is 55.7175, the validation accuracy is 0.976\n",
      "after 8100 training steps, the loss is 54.6234, the validation accuracy is 0.9794\n",
      "after 8200 training steps, the loss is 53.5026, the validation accuracy is 0.9792\n",
      "after 8300 training steps, the loss is 52.4529, the validation accuracy is 0.9802\n",
      "after 8400 training steps, the loss is 51.4203, the validation accuracy is 0.9804\n",
      "after 8500 training steps, the loss is 50.3953, the validation accuracy is 0.9792\n",
      "after 8600 training steps, the loss is 49.4201, the validation accuracy is 0.979\n",
      "after 8700 training steps, the loss is 48.4399, the validation accuracy is 0.9806\n",
      "after 8800 training steps, the loss is 47.4792, the validation accuracy is 0.9798\n",
      "after 8900 training steps, the loss is 46.5533, the validation accuracy is 0.9792\n",
      "after 9000 training steps, the loss is 45.6015, the validation accuracy is 0.9788\n",
      "after 9100 training steps, the loss is 44.7059, the validation accuracy is 0.98\n",
      "after 9200 training steps, the loss is 43.857, the validation accuracy is 0.978\n",
      "after 9300 training steps, the loss is 42.9802, the validation accuracy is 0.9816\n",
      "after 9400 training steps, the loss is 42.1064, the validation accuracy is 0.9804\n",
      "after 9500 training steps, the loss is 41.2916, the validation accuracy is 0.9798\n",
      "after 9600 training steps, the loss is 40.4812, the validation accuracy is 0.9802\n",
      "after 9700 training steps, the loss is 39.6988, the validation accuracy is 0.9784\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 9800 training steps, the loss is 38.8822, the validation accuracy is 0.9794\n",
      "after 9900 training steps, the loss is 38.1308, the validation accuracy is 0.9802\n",
      "after 10000 training steps, the loss is 37.3699, the validation accuracy is 0.9812\n",
      "after 10100 training steps, the loss is 36.6399, the validation accuracy is 0.9798\n",
      "after 10200 training steps, the loss is 35.9052, the validation accuracy is 0.979\n",
      "after 10300 training steps, the loss is 35.2024, the validation accuracy is 0.9814\n",
      "after 10400 training steps, the loss is 34.5162, the validation accuracy is 0.9802\n",
      "after 10500 training steps, the loss is 33.8388, the validation accuracy is 0.9804\n",
      "after 10600 training steps, the loss is 33.1851, the validation accuracy is 0.981\n",
      "after 10700 training steps, the loss is 32.5003, the validation accuracy is 0.9814\n",
      "after 10800 training steps, the loss is 31.8876, the validation accuracy is 0.9808\n",
      "after 10900 training steps, the loss is 31.2301, the validation accuracy is 0.9804\n",
      "after 11000 training steps, the loss is 30.6219, the validation accuracy is 0.9794\n",
      "after 11100 training steps, the loss is 30.0092, the validation accuracy is 0.9806\n",
      "after 11200 training steps, the loss is 29.4635, the validation accuracy is 0.9778\n",
      "after 11300 training steps, the loss is 28.8338, the validation accuracy is 0.9798\n",
      "after 11400 training steps, the loss is 28.2778, the validation accuracy is 0.982\n",
      "after 11500 training steps, the loss is 27.7538, the validation accuracy is 0.979\n",
      "after 11600 training steps, the loss is 27.165, the validation accuracy is 0.9812\n",
      "after 11700 training steps, the loss is 26.6934, the validation accuracy is 0.9792\n",
      "after 11800 training steps, the loss is 26.0927, the validation accuracy is 0.9814\n",
      "after 11900 training steps, the loss is 25.5893, the validation accuracy is 0.9812\n",
      "after 12000 training steps, the loss is 25.125, the validation accuracy is 0.9812\n",
      "after 12100 training steps, the loss is 24.659, the validation accuracy is 0.9816\n",
      "after 12200 training steps, the loss is 24.1037, the validation accuracy is 0.9818\n",
      "after 12300 training steps, the loss is 23.6588, the validation accuracy is 0.9804\n",
      "after 12400 training steps, the loss is 23.1817, the validation accuracy is 0.982\n",
      "after 12500 training steps, the loss is 22.7429, the validation accuracy is 0.9822\n",
      "after 12600 training steps, the loss is 22.2833, the validation accuracy is 0.9764\n",
      "after 12700 training steps, the loss is 21.8231, the validation accuracy is 0.9822\n",
      "after 12800 training steps, the loss is 21.3997, the validation accuracy is 0.9812\n",
      "after 12900 training steps, the loss is 20.9656, the validation accuracy is 0.9816\n",
      "after 13000 training steps, the loss is 20.5741, the validation accuracy is 0.9816\n",
      "after 13100 training steps, the loss is 20.1574, the validation accuracy is 0.9812\n",
      "after 13200 training steps, the loss is 19.7589, the validation accuracy is 0.982\n",
      "after 13300 training steps, the loss is 19.3857, the validation accuracy is 0.9806\n",
      "after 13400 training steps, the loss is 18.9873, the validation accuracy is 0.98\n",
      "after 13500 training steps, the loss is 18.6273, the validation accuracy is 0.9828\n",
      "after 13600 training steps, the loss is 18.2468, the validation accuracy is 0.9808\n",
      "after 13700 training steps, the loss is 17.9145, the validation accuracy is 0.9826\n",
      "after 13800 training steps, the loss is 17.5462, the validation accuracy is 0.9824\n",
      "after 13900 training steps, the loss is 17.1834, the validation accuracy is 0.9818\n",
      "after 14000 training steps, the loss is 16.8686, the validation accuracy is 0.9808\n",
      "after 14100 training steps, the loss is 16.5311, the validation accuracy is 0.9812\n",
      "after 14200 training steps, the loss is 16.1969, the validation accuracy is 0.981\n",
      "after 14300 training steps, the loss is 15.8832, the validation accuracy is 0.9824\n",
      "after 14400 training steps, the loss is 15.5632, the validation accuracy is 0.981\n",
      "after 14500 training steps, the loss is 15.2589, the validation accuracy is 0.982\n",
      "after 14600 training steps, the loss is 15.0554, the validation accuracy is 0.981\n",
      "after 14700 training steps, the loss is 14.7033, the validation accuracy is 0.982\n",
      "after 14800 training steps, the loss is 14.3902, the validation accuracy is 0.981\n",
      "after 14900 training steps, the loss is 14.133, the validation accuracy is 0.9814\n",
      "after 15000 training steps, the loss is 13.8265, the validation accuracy is 0.9794\n",
      "after 15100 training steps, the loss is 13.5675, the validation accuracy is 0.9814\n",
      "after 15200 training steps, the loss is 13.2776, the validation accuracy is 0.9814\n",
      "after 15300 training steps, the loss is 13.0175, the validation accuracy is 0.9828\n",
      "after 15400 training steps, the loss is 12.7711, the validation accuracy is 0.9822\n",
      "after 15500 training steps, the loss is 12.5046, the validation accuracy is 0.9802\n",
      "after 15600 training steps, the loss is 12.2839, the validation accuracy is 0.9802\n",
      "after 15700 training steps, the loss is 12.0385, the validation accuracy is 0.9816\n",
      "after 15800 training steps, the loss is 11.8037, the validation accuracy is 0.9826\n",
      "after 15900 training steps, the loss is 11.6041, the validation accuracy is 0.9834\n",
      "after 16000 training steps, the loss is 11.3412, the validation accuracy is 0.9798\n",
      "after 16100 training steps, the loss is 11.1257, the validation accuracy is 0.9802\n",
      "after 16200 training steps, the loss is 10.9096, the validation accuracy is 0.9826\n",
      "after 16300 training steps, the loss is 10.6735, the validation accuracy is 0.9816\n",
      "after 16400 training steps, the loss is 10.4849, the validation accuracy is 0.982\n",
      "after 16500 training steps, the loss is 10.2675, the validation accuracy is 0.9818\n",
      "after 16600 training steps, the loss is 10.1064, the validation accuracy is 0.982\n",
      "after 16700 training steps, the loss is 9.86881, the validation accuracy is 0.9818\n",
      "after 16800 training steps, the loss is 9.69513, the validation accuracy is 0.982\n",
      "after 16900 training steps, the loss is 9.4967, the validation accuracy is 0.9828\n",
      "after 17000 training steps, the loss is 9.30846, the validation accuracy is 0.9828\n",
      "after 17100 training steps, the loss is 9.13169, the validation accuracy is 0.9824\n",
      "after 17200 training steps, the loss is 8.932, the validation accuracy is 0.9824\n",
      "after 17300 training steps, the loss is 8.75211, the validation accuracy is 0.982\n",
      "after 17400 training steps, the loss is 8.6085, the validation accuracy is 0.9812\n",
      "after 17500 training steps, the loss is 8.42238, the validation accuracy is 0.9834\n",
      "after 17600 training steps, the loss is 8.25811, the validation accuracy is 0.9826\n",
      "after 17700 training steps, the loss is 8.1096, the validation accuracy is 0.9826\n",
      "after 17800 training steps, the loss is 7.94181, the validation accuracy is 0.9842\n",
      "after 17900 training steps, the loss is 7.79011, the validation accuracy is 0.9836\n",
      "after 18000 training steps, the loss is 7.6501, the validation accuracy is 0.9822\n",
      "after 18100 training steps, the loss is 7.56109, the validation accuracy is 0.9806\n",
      "after 18200 training steps, the loss is 7.34485, the validation accuracy is 0.984\n",
      "after 18300 training steps, the loss is 7.1891, the validation accuracy is 0.9826\n",
      "after 18400 training steps, the loss is 7.05801, the validation accuracy is 0.9836\n",
      "after 18500 training steps, the loss is 6.94454, the validation accuracy is 0.982\n",
      "after 18600 training steps, the loss is 6.79752, the validation accuracy is 0.9822\n",
      "after 18700 training steps, the loss is 6.6399, the validation accuracy is 0.983\n",
      "after 18800 training steps, the loss is 6.5142, the validation accuracy is 0.9816\n",
      "after 18900 training steps, the loss is 6.40662, the validation accuracy is 0.9834\n",
      "after 19000 training steps, the loss is 6.28798, the validation accuracy is 0.9832\n",
      "after 19100 training steps, the loss is 6.17546, the validation accuracy is 0.9816\n",
      "after 19200 training steps, the loss is 6.03698, the validation accuracy is 0.9816\n",
      "after 19300 training steps, the loss is 5.92129, the validation accuracy is 0.9816\n",
      "after 19400 training steps, the loss is 5.79593, the validation accuracy is 0.9822\n",
      "after 19500 training steps, the loss is 5.68376, the validation accuracy is 0.9812\n",
      "after 19600 training steps, the loss is 5.57356, the validation accuracy is 0.9816\n",
      "after 19700 training steps, the loss is 5.48223, the validation accuracy is 0.9812\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 19800 training steps, the loss is 5.38439, the validation accuracy is 0.9828\n",
      "after 19900 training steps, the loss is 5.24765, the validation accuracy is 0.9822\n",
      "the training is finish!\n",
      "the test accuarcy is: 0.9803\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "        #定义验证集与测试集\n",
    "    validate_data = {\n",
    "        x: mnist.validation.images,\n",
    "        y: mnist.validation.labels,\n",
    "    }\n",
    "    test_data = {x: mnist.test.images, y: mnist.test.labels}\n",
    "\n",
    "    for i in range(trainig_step):\n",
    "        xs, ys = mnist.train.next_batch(batch_size)\n",
    "        _, loss_value = sess.run(\n",
    "            [optimizer, loss],\n",
    "            feed_dict={\n",
    "                x: xs,\n",
    "                y: ys,\n",
    "                learning_rate: 0.1\n",
    "            })\n",
    "\n",
    "        #每100次训练打印一次损失值与验证准确率\n",
    "        if i > 0 and i % 100 == 0:\n",
    "            validate_accuracy = sess.run(accuracy, feed_dict=validate_data)\n",
    "            print(\n",
    "                \"after %d training steps, the loss is %g, the validation accuracy is %g\"\n",
    "                % (i, loss_value, validate_accuracy))\n",
    "            saver.save(sess, './model.ckpt', global_step=i)\n",
    "\n",
    "    print(\"the training is finish!\")\n",
    "    #最终的测试准确率\n",
    "    acc = sess.run(accuracy, feed_dict=test_data)\n",
    "    print(\"the test accuarcy is:\", acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "l2正则比l1正则效果好。增加了正则虽然缓解了过拟合，训练轮数也要相应增加，收敛难度也增加了。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def initialize(shape, stddev=0.1):#定义初始化函数。\n",
    "  return tf.truncated_normal(shape, stddev=0.1)#用truncated_normal函数进行初始化，truncated_normal是截断正态分布，会删除大于2个stddev的x值"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
